From 7bd9710384933097bc573c72ad0f5f453c97402c Mon Sep 17 00:00:00 2001 From: igutierrez Date: Fri, 6 Nov 2026 12:21:37 -0600 Subject: [PATCH 01/35] Publish the kumo-data-science skill to the NVIDIA skills catalog NVIDIA/skills pulls skills from product repos every hour. It accepts only content that NVSkills CI has signed, and NVSkills CI runs only on PRs that touch a root-level skills/ directory. The source layout in agent-skills/skills/ is not a package, so the catalog cannot read it directly. - agent-skills/scripts/publish_nvskills.py writes the generic package to skills/kumo-data-science/, copies in the catalog files from agent-skills/nvskills/, and leaves the files NVSkills CI wrote in place. --check reports drift and ignores the SKILL-BUILD stamp. - nvskills-publish.yml runs it on every skill change to main and opens or updates one PR from nvskills/kumo-data-science. - request-nvskills-ci.yml is NVIDIA/skills' request workflow, pinned to a SHA as the workflow-security policy requires. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- .github/workflows/nvskills-publish.yml | 89 ++++++++++++ .github/workflows/request-nvskills-ci.yml | 36 +++++ agent-skills/README.md | 23 ++++ agent-skills/scripts/publish_nvskills.py | 143 ++++++++++++++++++++ agent-skills/tests/test_publish_nvskills.py | 70 ++++++++++ 5 files changed, 361 insertions(+) create mode 100644 .github/workflows/nvskills-publish.yml create mode 100644 .github/workflows/request-nvskills-ci.yml create mode 100644 agent-skills/scripts/publish_nvskills.py create mode 100644 agent-skills/tests/test_publish_nvskills.py diff --git a/.github/workflows/nvskills-publish.yml b/.github/workflows/nvskills-publish.yml new file mode 100644 index 000000000..ab594a863 --- /dev/null +++ b/.github/workflows/nvskills-publish.yml @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Publish skill to NVIDIA/skills + +# The NVIDIA/skills catalog syncs skills/kumo-data-science/ from main every hour, but only +# content that NVSkills CI has signed. When the skill source changes on main, this workflow +# rebuilds the published copy and opens or updates one PR on a bot branch. A maintainer then +# comments `/nvskills-ci` on that PR. NVSkills CI runs the evals and pushes BENCHMARK.md and +# skill.oms.sig. Once that commit is on the PR, it can merge. +# +# A PR opened with GITHUB_TOKEN does not trigger other workflows, so this repository's own CI +# does not run on it. The `/nvskills-ci` comment and the signing push do trigger +# request-nvskills-ci.yml, because a person and nv-skills-ci[bot] make them. The PR changes +# only skills/, which is generated output that the Agent Skills workflow already tested on main. + +on: # yamllint disable-line rule:truthy + push: + branches: [main] + paths: + - "agent-skills/skills/**" + - "agent-skills/nvskills/**" + - "agent-skills/scripts/package_skill.py" + - "agent-skills/scripts/publish_nvskills.py" + - ".github/workflows/nvskills-publish.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nvskills-publish + cancel-in-progress: false + +jobs: + open-pr: + if: github.ref_name == 'main' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: nvskills/kumo-data-science + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Rebuild skills/kumo-data-science/ + run: python3 agent-skills/scripts/publish_nvskills.py | tee /tmp/publish.txt + + - name: Commit and open or update the PR + run: | + set -euo pipefail + if [ -z "$(git status --porcelain -- skills/)" ]; then + echo "skills/kumo-data-science/ already matches main; nothing to publish." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + short="$(git rev-parse --short=8 HEAD)" + git checkout -B "$BRANCH" + git add skills/ + git commit -q -m "Publish kumo-data-science skill from ${short}" + # The branch is owned by this workflow. Content built from a newer main replaces it. Any + # signatures already on the branch covered the older content, so NVSkills CI re-runs. + git push --force origin "$BRANCH" + + { + echo "Rebuilds \`skills/kumo-data-science/\` from \`agent-skills/\` at ${GITHUB_SHA}." + echo "The NVIDIA/skills catalog syncs this directory every hour." + echo + echo "**To publish:** a maintainer comments \`/nvskills-ci\` here. NVSkills CI runs the" + echo "skill's evals and pushes \`BENCHMARK.md\` and \`skill.oms.sig\`. Merge once that" + echo "commit lands. If this PR merges without it, the catalog keeps the last signed version." + echo + echo '```' + cat /tmp/publish.txt + echo '```' + } > /tmp/body.md + + if [ "$(gh pr view "$BRANCH" --json state --jq .state 2>/dev/null)" = "OPEN" ]; then + gh pr edit "$BRANCH" --body-file /tmp/body.md + else + gh pr create --base main --head "$BRANCH" \ + --title "Publish kumo-data-science skill to NVIDIA/skills" \ + --body-file /tmp/body.md + fi diff --git a/.github/workflows/request-nvskills-ci.yml b/.github/workflows/request-nvskills-ci.yml new file mode 100644 index 000000000..9701cfe74 --- /dev/null +++ b/.github/workflows/request-nvskills-ci.yml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# NVSkills CI: evaluates and signs changes under skills/, which the NVIDIA/skills catalog syncs +# from. This is NVIDIA/skills' request workflow, unchanged except for the SHA pin, which this +# repository's workflow-security policy requires. When NVIDIA/skills changes the template, re-pin +# to its current main. A maintainer or admin comments `/nvskills-ci` on a PR to start a run. +# Requires the NVSKILLS_CI_DISPATCH_TOKEN secret. PRs that do not touch skills/ skip it. + +name: Request NVSkills CI + +on: # yamllint disable-line rule:truthy + issue_comment: + types: [created] + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + push: + +jobs: + request: + if: > + github.event_name == 'pull_request' || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/nvskills-ci')) || + (github.event_name == 'push' && + github.actor == (vars.NVSKILLS_SIGNATURE_PUSH_ACTOR || 'nv-skills-ci[bot]') && + startsWith(github.event.head_commit.message, vars.NVSKILLS_SIGNATURE_COMMIT_TITLE || 'Attach NVSkills validation signatures')) + permissions: + contents: read + pull-requests: read + statuses: read + uses: NVIDIA/skills/.github/workflows/team-request.yml@b707bbe61ff1026dd59a44140610f92c8eae1151 # main, 2026-11-06 + secrets: + NVSKILLS_CI_DISPATCH_TOKEN: ${{ secrets.NVSKILLS_CI_DISPATCH_TOKEN }} diff --git a/agent-skills/README.md b/agent-skills/README.md index 65726650a..530b5887a 100644 --- a/agent-skills/README.md +++ b/agent-skills/README.md @@ -125,3 +125,26 @@ The skills cite Kumo code by module path, as the agent sees it in a notebook resolves each one against `packages/`, and checks every `kumo.` against `kumo.engine.__all__`, so the workflow also runs on changes to `kumo-engine`, `kumo-core`, `kumo-api` and `kumo-snowflake`. `kumoml.*` lives outside this repository and is not checked. + +## Publishing to the NVIDIA skills catalog + +[NVIDIA/skills](https://github.com/NVIDIA/skills) pulls every product's skills from its source repo +once an hour. It accepts only content that NVSkills CI has evaluated and signed. NVSkills CI runs +only on PRs that change a root-level `skills/` directory. So the catalog reads a built copy, +`skills/kumo-data-science/` at the repository root, not `agent-skills/skills/`: + +| Path | Written by | +|---|---| +| `skills/kumo-data-science/` (everything else) | `agent-skills/scripts/publish_nvskills.py`: the `generic` package | +| `skills/kumo-data-science/skill-card.md`, `evals/` | copied from `agent-skills/nvskills/kumo-data-science/` | +| `skills/kumo-data-science/BENCHMARK.md`, `skill.oms.sig` | NVSkills CI, in the publish PR; never by hand | + +Do not edit `skills/` directly. Change `agent-skills/` and merge. The `nvskills-publish` workflow +rebuilds the copy and opens or updates one PR from `nvskills/kumo-data-science`. A maintainer +comments `/nvskills-ci` on that PR. NVSkills CI runs the evals and pushes the signature commit, and +then the PR can merge. `python3 agent-skills/scripts/publish_nvskills.py --check` reports whether +the copy on your branch is out of date. It ignores the `SKILL-BUILD` stamp. + +The catalog side is `components.d/kumo-engine.yml` in NVIDIA/skills, which points at +`skills/kumo-data-science`. Its `verify-source-onboarding` check fails until a signed copy with +`skill-card.md`, `BENCHMARK.md` and an eval dataset is on `main` here. diff --git a/agent-skills/scripts/publish_nvskills.py b/agent-skills/scripts/publish_nvskills.py new file mode 100644 index 000000000..53f364b99 --- /dev/null +++ b/agent-skills/scripts/publish_nvskills.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Refresh the copy of the skill that the NVIDIA skills catalog publishes. + + python3 agent-skills/scripts/publish_nvskills.py # rewrite skills/kumo-data-science/ + python3 agent-skills/scripts/publish_nvskills.py --check # exit 1 if that copy is out of date + +The NVIDIA/skills catalog (github.com/NVIDIA/skills) pulls skills from each product repo +every hour. It reads them from `components.d/.yml` in the catalog, which names a +directory in this repository. It accepts a skill only if the skill's files match the signature +that NVSkills CI writes. NVSkills CI runs only on pull requests that change `skills/` or +`team-skills/` at the repository root. `agent-skills/skills/` is the source layout, not a +package, so the catalog cannot read it directly. This script writes the `generic` package (the one +for Claude Code, Codex and other coding agents) to `skills/kumo-data-science/`. The +`nvskills-publish` workflow runs it on every change to `main` and opens a pull request with +the result. NVSkills CI then signs that pull request. + +Three kinds of file sit in the published directory: + +* the package, rebuilt from `agent-skills/skills/` on every run; +* the catalog files that NVSkills requires and that we write: `skill-card.md` and + `evals/*.json`. Their source is `agent-skills/nvskills/kumo-data-science/`, and they are + copied in on every run; +* `BENCHMARK.md` and `skill.oms.sig`, which NVSkills CI writes into the pull request. They are + never edited here. A refresh leaves them in place, so they describe the previous content until + NVSkills CI runs again. The catalog then keeps serving the last signed version rather than + dropping the skill. + +`--check` ignores the `SKILL-BUILD` stamp, which names the commit that built the package and so +differs on every commit even when no skill file changed. +""" +from __future__ import annotations + +import argparse +import importlib.util +import shutil +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('package_skill', HERE / 'package_skill.py') +assert spec and spec.loader +package_skill = importlib.util.module_from_spec(spec) +sys.modules.setdefault('package_skill', package_skill) +spec.loader.exec_module(package_skill) + +HOST = 'generic' +REPO = package_skill.ROOT.parent +PUBLISHED = REPO / 'skills' / package_skill.ROUTER +CATALOG_FILES = package_skill.ROOT / 'nvskills' / package_skill.ROUTER +# Written by NVSkills CI into the pull request; never generated or deleted here. +SIGNED = frozenset({'BENCHMARK.md', 'skill.oms.sig'}) + + +def build(out: Path, catalog_files: Path = CATALOG_FILES) -> Path: + """Build the package the catalog publishes into `out` and return its skill directory.""" + pkg = package_skill.build(HOST, out) + errors = package_skill.unresolved(pkg) + if errors: + raise SystemExit(f'{HOST}: {len(errors)} unresolved reference(s): {errors[:3]}') + if catalog_files.is_dir(): + for path in sorted(catalog_files.rglob('*')): + if path.is_file(): + rel = path.relative_to(catalog_files) + if rel.as_posix() in SIGNED: + raise SystemExit(f'{path}: {rel} is written by NVSkills CI, not by hand') + target = pkg / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + return pkg + + +def files(root: Path) -> dict[str, bytes]: + """Map each file under `root` that we own to its bytes, with the build stamp normalized.""" + if not root.is_dir(): + return {} + out = {} + for path in sorted(root.rglob('*')): + rel = path.relative_to(root).as_posix() + if path.is_file() and rel not in SIGNED: + data = path.read_bytes() + if rel == 'SKILL.md': + text = package_skill.MARKER.sub('SKILL-BUILD: x', data.decode('utf-8')) + data = text.encode('utf-8') + out[rel] = data + return out + + +def stale(dest: Path = PUBLISHED, catalog_files: Path = CATALOG_FILES) -> list[str]: + """Return what `dest` would gain, lose or change if it were refreshed now.""" + with tempfile.TemporaryDirectory() as tmp: + want = files(build(Path(tmp), catalog_files)) + have = files(dest) + return ([f'+ {p}' for p in sorted(want.keys() - have.keys())] + + [f'- {p}' for p in sorted(have.keys() - want.keys())] + + [f'~ {p}' for p in sorted(want.keys() & have.keys()) if want[p] != have[p]]) + + +def publish(dest: Path = PUBLISHED, catalog_files: Path = CATALOG_FILES) -> list[str]: + """Rewrite `dest` with the current package, keeping the files NVSkills CI wrote.""" + changes = stale(dest, catalog_files) + if not changes: + return [] + with tempfile.TemporaryDirectory() as tmp: + pkg = build(Path(tmp), catalog_files) + if dest.is_dir(): + for path in sorted(dest.rglob('*'), reverse=True): + rel = path.relative_to(dest).as_posix() + if path.is_file() and rel not in SIGNED: + path.unlink() + elif path.is_dir() and not any(path.iterdir()): + path.rmdir() + for path in sorted(pkg.rglob('*')): + if path.is_file(): + target = dest / path.relative_to(pkg) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + return changes + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('--check', action='store_true', + help=f'exit 1 if {PUBLISHED.relative_to(REPO)}/ is out of date') + args = ap.parse_args() + where = PUBLISHED.relative_to(REPO) + changes = stale() if args.check else publish() + for line in changes: + print(line) + if not changes: + print(f'{where}/ is up to date') + elif args.check: + print(f'{where}/ is out of date: run {Path(__file__).relative_to(REPO)}', file=sys.stderr) + raise SystemExit(1) + else: + print(f'{where}/ refreshed: {len(changes)} file(s)') + + +if __name__ == '__main__': + main() diff --git a/agent-skills/tests/test_publish_nvskills.py b/agent-skills/tests/test_publish_nvskills.py new file mode 100644 index 000000000..c0f57ede6 --- /dev/null +++ b/agent-skills/tests/test_publish_nvskills.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import importlib.util +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / 'scripts' / 'publish_nvskills.py' +spec = importlib.util.spec_from_file_location('publish_nvskills', SCRIPT) +assert spec and spec.loader +publish_nvskills = importlib.util.module_from_spec(spec) +sys.modules['publish_nvskills'] = publish_nvskills +spec.loader.exec_module(publish_nvskills) + + +@pytest.fixture +def catalog(tmp_path: Path) -> Path: + root = tmp_path / 'catalog' + (root / 'evals').mkdir(parents=True) + (root / 'evals' / 'evals.json').write_text('{"skill_name": "kumo-data-science"}\n') + (root / 'skill-card.md').write_text('## Description\n') + return root + + +def test_publish_writes_the_generic_package_and_catalog_files(tmp_path: Path, + catalog: Path) -> None: + dest = tmp_path / 'skills' / 'kumo-data-science' + assert publish_nvskills.publish(dest, catalog) + assert (dest / 'SKILL.md').is_file() + assert (dest / 'hosts' / 'generic' / 'adapter.md').is_file() + assert not (dest / 'hosts' / 'snowflake-coco').exists() + assert (dest / 'evals' / 'evals.json').is_file() + assert (dest / 'skill-card.md').is_file() + assert publish_nvskills.stale(dest, catalog) == [] + + +def test_publish_keeps_signed_files_and_drops_files_no_longer_shipped(tmp_path: Path, + catalog: Path) -> None: + dest = tmp_path / 'kumo-data-science' + publish_nvskills.publish(dest, catalog) + for name in publish_nvskills.SIGNED: + (dest / name).write_text('from NVSkills CI\n') + (dest / 'old' / 'removed.md').parent.mkdir() + (dest / 'old' / 'removed.md').write_text('gone\n') + + assert publish_nvskills.stale(dest, catalog) == ['- old/removed.md'] + publish_nvskills.publish(dest, catalog) + assert not (dest / 'old').exists() + for name in publish_nvskills.SIGNED: + assert (dest / name).read_text() == 'from NVSkills CI\n' + + +def test_check_ignores_the_build_stamp_but_not_content(tmp_path: Path, catalog: Path) -> None: + dest = tmp_path / 'kumo-data-science' + publish_nvskills.publish(dest, catalog) + router = dest / 'SKILL.md' + text = router.read_text() + router.write_text(publish_nvskills.package_skill.MARKER.sub('SKILL-BUILD: 2000-01-01-0', text)) + assert publish_nvskills.stale(dest, catalog) == [] + + router.write_text(text + '\nedited by hand\n') + assert publish_nvskills.stale(dest, catalog) == ['~ SKILL.md'] + + +def test_catalog_files_cannot_supply_signed_files(tmp_path: Path, catalog: Path) -> None: + (catalog / 'skill.oms.sig').write_text('forged\n') + with pytest.raises(SystemExit, match='written by NVSkills CI'): + publish_nvskills.publish(tmp_path / 'kumo-data-science', catalog) From 113fde01ee461d0731a6907a65f4aaf19e242b32 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Fri, 6 Nov 2026 14:00:58 -0600 Subject: [PATCH 02/35] nvskills-publish: post the change map and name /ok to test The bot PR fails the PR change table check without a change-map comment. github-actions[bot] may post it, so the workflow writes one row per changed file and edits that one comment on later pushes. The required kumo-core-gpu-ci check runs through copy-pr-bot, which does not trust the workflow's unsigned commit, so the PR body tells the maintainer to comment /ok to test . Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- .github/workflows/nvskills-publish.yml | 34 +++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nvskills-publish.yml b/.github/workflows/nvskills-publish.yml index ab594a863..b00191366 100644 --- a/.github/workflows/nvskills-publish.yml +++ b/.github/workflows/nvskills-publish.yml @@ -14,6 +14,10 @@ name: Publish skill to NVIDIA/skills # does not run on it. The `/nvskills-ci` comment and the signing push do trigger # request-nvskills-ci.yml, because a person and nv-skills-ci[bot] make them. The PR changes # only skills/, which is generated output that the Agent Skills workflow already tested on main. +# +# The required kumo-core-gpu-ci check runs through copy-pr-bot. It does not trust this +# workflow's unsigned commit, so a maintainer also comments `/ok to test `. The +# change-map comment that the PR change table check reads is posted here. on: # yamllint disable-line rule:truthy push: @@ -67,13 +71,15 @@ jobs: # signatures already on the branch covered the older content, so NVSkills CI re-runs. git push --force origin "$BRANCH" + head="$(git rev-parse HEAD)" { echo "Rebuilds \`skills/kumo-data-science/\` from \`agent-skills/\` at ${GITHUB_SHA}." echo "The NVIDIA/skills catalog syncs this directory every hour." echo - echo "**To publish:** a maintainer comments \`/nvskills-ci\` here. NVSkills CI runs the" - echo "skill's evals and pushes \`BENCHMARK.md\` and \`skill.oms.sig\`. Merge once that" - echo "commit lands. If this PR merges without it, the catalog keeps the last signed version." + echo "**To publish:** a maintainer comments \`/ok to test ${head}\` to run the required" + echo "GPU check, and \`/nvskills-ci\` to have NVSkills CI run the skill's evals and push" + echo "\`BENCHMARK.md\` and \`skill.oms.sig\`. Merge once that commit lands. If this PR" + echo "merges without it, the catalog keeps the last signed version." echo echo '```' cat /tmp/publish.txt @@ -87,3 +93,25 @@ jobs: --title "Publish kumo-data-science skill to NVIDIA/skills" \ --body-file /tmp/body.md fi + + # The change map the PR change table check reads: one comment, edited on each push. + marker='' + { + echo "$marker" + echo "## Change map" + echo + echo "| File | What changed | Why | Function / scope |" + echo "| --- | --- | --- | --- |" + git diff --name-status HEAD~1 HEAD | while IFS=$'\t' read -r status path; do + case "$status" in A) what=Added ;; D) what=Deleted ;; *) what=Rebuilt ;; esac + echo "| \`$path\` | $what | Published package of \`agent-skills/\` at ${short} | kumo-data-science skill for NVIDIA/skills |" + done + } > /tmp/map.md + number="$(gh pr view "$BRANCH" --json number --jq .number)" + comment="$(gh api "repos/${GITHUB_REPOSITORY}/issues/${number}/comments" --paginate \ + --jq "[.[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\")))][0].id // empty")" + if [ -n "$comment" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${comment}" -F body=@/tmp/map.md >/dev/null + else + gh pr comment "$number" --body-file /tmp/map.md + fi From 49ff6640eb5546633daa1d345b6a409e382949f1 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Fri, 6 Nov 2026 14:29:36 -0600 Subject: [PATCH 03/35] Publish from main: commit the catalog copy with each skill change NVSkills CI signs only PRs, and main needs an approval and the GPU check, which a bot PR cannot give itself. So the published copy now travels in the same PR as the change it comes from, and merging that PR is what publishes. - skills/kumo-data-science/ is committed: the generic package, built by publish_nvskills.py. - test_the_published_copy_matches_the_source fails a PR that changes the skill without rebuilding the copy. agent-skills.yml now also runs on skills/**. - nvskills-publish.yml is removed, since nothing opens a bot PR any more. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- .github/workflows/agent-skills.yml | 4 + .github/workflows/nvskills-publish.yml | 117 --- agent-skills/README.md | 19 +- agent-skills/scripts/publish_nvskills.py | 6 +- agent-skills/tests/test_publish_nvskills.py | 13 + skills/kumo-data-science/SKILL.md | 330 +++++++ .../accept/batch_inference.md | 191 ++++ .../accept/intake_contract.md | 177 ++++ skills/kumo-data-science/accept/preflight.md | 225 +++++ .../kumo-data-science/accept/semantic_view.md | 146 +++ .../hosts/generic/adapter.md | 149 +++ .../hosts/generic/runtime.md | 325 +++++++ .../kumo-custom-populations/SKILL.md | 20 + .../references/populations.md | 159 ++++ .../kumo-graph-visualization/SKILL.md | 77 ++ .../kumo-model-explanations/SKILL.md | 84 ++ .../references/artifacts.md | 287 ++++++ .../kumo-model-persistence/SKILL.md | 25 + .../references/retention.md | 87 ++ .../references/reuse.md | 117 +++ .../scripts/retain_artifacts.py | 142 +++ .../kumo-model-tuning/SKILL.md | 22 + .../kumo-model-tuning/references/plans.md | 82 ++ .../kumo-training-diagnostics/SKILL.md | 79 ++ .../references/diagnostics.md | 179 ++++ .../kumoai-reference/SKILL.md | 98 ++ .../references/graph_construction.md | 409 ++++++++ .../references/kumo_finetuning_sdk.md | 879 ++++++++++++++++++ .../kumoai-reference/references/outputs.md | 100 ++ .../references/pql_writing.md | 502 ++++++++++ .../references/task_journeys.md | 111 +++ .../references/troubleshooting_signals.md | 274 ++++++ .../validate/atomic_question_check.md | 84 ++ .../validate/examples/bad_request.md | 68 ++ .../validate/examples/good_request.md | 106 +++ 35 files changed, 5566 insertions(+), 127 deletions(-) delete mode 100644 .github/workflows/nvskills-publish.yml create mode 100644 skills/kumo-data-science/SKILL.md create mode 100644 skills/kumo-data-science/accept/batch_inference.md create mode 100644 skills/kumo-data-science/accept/intake_contract.md create mode 100644 skills/kumo-data-science/accept/preflight.md create mode 100644 skills/kumo-data-science/accept/semantic_view.md create mode 100644 skills/kumo-data-science/hosts/generic/adapter.md create mode 100644 skills/kumo-data-science/hosts/generic/runtime.md create mode 100644 skills/kumo-data-science/kumo-custom-populations/SKILL.md create mode 100644 skills/kumo-data-science/kumo-custom-populations/references/populations.md create mode 100644 skills/kumo-data-science/kumo-graph-visualization/SKILL.md create mode 100644 skills/kumo-data-science/kumo-model-explanations/SKILL.md create mode 100644 skills/kumo-data-science/kumo-model-explanations/references/artifacts.md create mode 100644 skills/kumo-data-science/kumo-model-persistence/SKILL.md create mode 100644 skills/kumo-data-science/kumo-model-persistence/references/retention.md create mode 100644 skills/kumo-data-science/kumo-model-persistence/references/reuse.md create mode 100644 skills/kumo-data-science/kumo-model-persistence/scripts/retain_artifacts.py create mode 100644 skills/kumo-data-science/kumo-model-tuning/SKILL.md create mode 100644 skills/kumo-data-science/kumo-model-tuning/references/plans.md create mode 100644 skills/kumo-data-science/kumo-training-diagnostics/SKILL.md create mode 100644 skills/kumo-data-science/kumo-training-diagnostics/references/diagnostics.md create mode 100644 skills/kumo-data-science/kumoai-reference/SKILL.md create mode 100644 skills/kumo-data-science/kumoai-reference/references/graph_construction.md create mode 100644 skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md create mode 100644 skills/kumo-data-science/kumoai-reference/references/outputs.md create mode 100644 skills/kumo-data-science/kumoai-reference/references/pql_writing.md create mode 100644 skills/kumo-data-science/kumoai-reference/references/task_journeys.md create mode 100644 skills/kumo-data-science/kumoai-reference/references/troubleshooting_signals.md create mode 100644 skills/kumo-data-science/validate/atomic_question_check.md create mode 100644 skills/kumo-data-science/validate/examples/bad_request.md create mode 100644 skills/kumo-data-science/validate/examples/good_request.md diff --git a/.github/workflows/agent-skills.yml b/.github/workflows/agent-skills.yml index 67548ce00..b3a65cc48 100644 --- a/.github/workflows/agent-skills.yml +++ b/.github/workflows/agent-skills.yml @@ -10,6 +10,8 @@ on: # yamllint disable-line rule:truthy - main paths: - "agent-skills/**" + # The copy NVIDIA/skills publishes; it must match agent-skills/ (publish_nvskills.py). + - "skills/**" # The skills cite modules and names from these packages; a rename must fail here. - "packages/kumo-engine/**" - "packages/kumo-core/**" @@ -20,6 +22,8 @@ on: # yamllint disable-line rule:truthy pull_request: paths: - "agent-skills/**" + # The copy NVIDIA/skills publishes; it must match agent-skills/ (publish_nvskills.py). + - "skills/**" # The skills cite modules and names from these packages; a rename must fail here. - "packages/kumo-engine/**" - "packages/kumo-core/**" diff --git a/.github/workflows/nvskills-publish.yml b/.github/workflows/nvskills-publish.yml deleted file mode 100644 index b00191366..000000000 --- a/.github/workflows/nvskills-publish.yml +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: Publish skill to NVIDIA/skills - -# The NVIDIA/skills catalog syncs skills/kumo-data-science/ from main every hour, but only -# content that NVSkills CI has signed. When the skill source changes on main, this workflow -# rebuilds the published copy and opens or updates one PR on a bot branch. A maintainer then -# comments `/nvskills-ci` on that PR. NVSkills CI runs the evals and pushes BENCHMARK.md and -# skill.oms.sig. Once that commit is on the PR, it can merge. -# -# A PR opened with GITHUB_TOKEN does not trigger other workflows, so this repository's own CI -# does not run on it. The `/nvskills-ci` comment and the signing push do trigger -# request-nvskills-ci.yml, because a person and nv-skills-ci[bot] make them. The PR changes -# only skills/, which is generated output that the Agent Skills workflow already tested on main. -# -# The required kumo-core-gpu-ci check runs through copy-pr-bot. It does not trust this -# workflow's unsigned commit, so a maintainer also comments `/ok to test `. The -# change-map comment that the PR change table check reads is posted here. - -on: # yamllint disable-line rule:truthy - push: - branches: [main] - paths: - - "agent-skills/skills/**" - - "agent-skills/nvskills/**" - - "agent-skills/scripts/package_skill.py" - - "agent-skills/scripts/publish_nvskills.py" - - ".github/workflows/nvskills-publish.yml" - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: nvskills-publish - cancel-in-progress: false - -jobs: - open-pr: - if: github.ref_name == 'main' - runs-on: ubuntu-24.04 - timeout-minutes: 10 - permissions: - contents: write - pull-requests: write - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: nvskills/kumo-data-science - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - - name: Rebuild skills/kumo-data-science/ - run: python3 agent-skills/scripts/publish_nvskills.py | tee /tmp/publish.txt - - - name: Commit and open or update the PR - run: | - set -euo pipefail - if [ -z "$(git status --porcelain -- skills/)" ]; then - echo "skills/kumo-data-science/ already matches main; nothing to publish." - exit 0 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - short="$(git rev-parse --short=8 HEAD)" - git checkout -B "$BRANCH" - git add skills/ - git commit -q -m "Publish kumo-data-science skill from ${short}" - # The branch is owned by this workflow. Content built from a newer main replaces it. Any - # signatures already on the branch covered the older content, so NVSkills CI re-runs. - git push --force origin "$BRANCH" - - head="$(git rev-parse HEAD)" - { - echo "Rebuilds \`skills/kumo-data-science/\` from \`agent-skills/\` at ${GITHUB_SHA}." - echo "The NVIDIA/skills catalog syncs this directory every hour." - echo - echo "**To publish:** a maintainer comments \`/ok to test ${head}\` to run the required" - echo "GPU check, and \`/nvskills-ci\` to have NVSkills CI run the skill's evals and push" - echo "\`BENCHMARK.md\` and \`skill.oms.sig\`. Merge once that commit lands. If this PR" - echo "merges without it, the catalog keeps the last signed version." - echo - echo '```' - cat /tmp/publish.txt - echo '```' - } > /tmp/body.md - - if [ "$(gh pr view "$BRANCH" --json state --jq .state 2>/dev/null)" = "OPEN" ]; then - gh pr edit "$BRANCH" --body-file /tmp/body.md - else - gh pr create --base main --head "$BRANCH" \ - --title "Publish kumo-data-science skill to NVIDIA/skills" \ - --body-file /tmp/body.md - fi - - # The change map the PR change table check reads: one comment, edited on each push. - marker='' - { - echo "$marker" - echo "## Change map" - echo - echo "| File | What changed | Why | Function / scope |" - echo "| --- | --- | --- | --- |" - git diff --name-status HEAD~1 HEAD | while IFS=$'\t' read -r status path; do - case "$status" in A) what=Added ;; D) what=Deleted ;; *) what=Rebuilt ;; esac - echo "| \`$path\` | $what | Published package of \`agent-skills/\` at ${short} | kumo-data-science skill for NVIDIA/skills |" - done - } > /tmp/map.md - number="$(gh pr view "$BRANCH" --json number --jq .number)" - comment="$(gh api "repos/${GITHUB_REPOSITORY}/issues/${number}/comments" --paginate \ - --jq "[.[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\")))][0].id // empty")" - if [ -n "$comment" ]; then - gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${comment}" -F body=@/tmp/map.md >/dev/null - else - gh pr comment "$number" --body-file /tmp/map.md - fi diff --git a/agent-skills/README.md b/agent-skills/README.md index 530b5887a..bea871dcc 100644 --- a/agent-skills/README.md +++ b/agent-skills/README.md @@ -137,13 +137,18 @@ only on PRs that change a root-level `skills/` directory. So the catalog reads a |---|---| | `skills/kumo-data-science/` (everything else) | `agent-skills/scripts/publish_nvskills.py`: the `generic` package | | `skills/kumo-data-science/skill-card.md`, `evals/` | copied from `agent-skills/nvskills/kumo-data-science/` | -| `skills/kumo-data-science/BENCHMARK.md`, `skill.oms.sig` | NVSkills CI, in the publish PR; never by hand | - -Do not edit `skills/` directly. Change `agent-skills/` and merge. The `nvskills-publish` workflow -rebuilds the copy and opens or updates one PR from `nvskills/kumo-data-science`. A maintainer -comments `/nvskills-ci` on that PR. NVSkills CI runs the evals and pushes the signature commit, and -then the PR can merge. `python3 agent-skills/scripts/publish_nvskills.py --check` reports whether -the copy on your branch is out of date. It ignores the `SKILL-BUILD` stamp. +| `skills/kumo-data-science/BENCHMARK.md`, `skill.oms.sig` | NVSkills CI, in your PR; never by hand | + +Whatever is on `main` is what the catalog publishes. To change the skill: + +1. Edit `agent-skills/`, never `skills/`. +2. Run `python3 agent-skills/scripts/publish_nvskills.py` and commit `skills/` with your change. + `tests/test_publish_nvskills.py` fails the PR if you forget. It ignores the `SKILL-BUILD` + stamp, so a rebuild on an unrelated commit is not required. +3. A maintainer comments `/nvskills-ci` on the PR. NVSkills CI runs the evals and pushes + `BENCHMARK.md` and `skill.oms.sig`. A later push to the PR needs another `/nvskills-ci`. +4. Merge. The catalog picks the change up within the hour. A PR merged without step 3 leaves the + catalog on the last signed version. The catalog side is `components.d/kumo-engine.yml` in NVIDIA/skills, which points at `skills/kumo-data-science`. Its `verify-source-onboarding` check fails until a signed copy with diff --git a/agent-skills/scripts/publish_nvskills.py b/agent-skills/scripts/publish_nvskills.py index 53f364b99..2d68f68b1 100644 --- a/agent-skills/scripts/publish_nvskills.py +++ b/agent-skills/scripts/publish_nvskills.py @@ -13,9 +13,9 @@ that NVSkills CI writes. NVSkills CI runs only on pull requests that change `skills/` or `team-skills/` at the repository root. `agent-skills/skills/` is the source layout, not a package, so the catalog cannot read it directly. This script writes the `generic` package (the one -for Claude Code, Codex and other coding agents) to `skills/kumo-data-science/`. The -`nvskills-publish` workflow runs it on every change to `main` and opens a pull request with -the result. NVSkills CI then signs that pull request. +for Claude Code, Codex and other coding agents) to `skills/kumo-data-science/`. A PR that changes +the skill runs it and commits the result, and `tests/test_publish_nvskills.py` fails the PR when +it did not. NVSkills CI signs that same PR, so merging it to `main` is what publishes. Three kinds of file sit in the published directory: diff --git a/agent-skills/tests/test_publish_nvskills.py b/agent-skills/tests/test_publish_nvskills.py index c0f57ede6..67f3bae0d 100644 --- a/agent-skills/tests/test_publish_nvskills.py +++ b/agent-skills/tests/test_publish_nvskills.py @@ -68,3 +68,16 @@ def test_catalog_files_cannot_supply_signed_files(tmp_path: Path, catalog: Path) (catalog / 'skill.oms.sig').write_text('forged\n') with pytest.raises(SystemExit, match='written by NVSkills CI'): publish_nvskills.publish(tmp_path / 'kumo-data-science', catalog) + + +def test_the_published_copy_matches_the_source() -> None: + """The copy NVIDIA/skills syncs from main is rebuilt in the same PR as the change. + + A PR that edits agent-skills/ without refreshing skills/kumo-data-science/ would merge a + published copy that no longer matches its source, so it fails here instead. + """ + changes = publish_nvskills.stale() + assert changes == [], ( + 'skills/kumo-data-science/ is out of date; run ' + f'python3 agent-skills/scripts/publish_nvskills.py and commit the result: {changes}' + ) diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md new file mode 100644 index 000000000..aab8c4229 --- /dev/null +++ b/skills/kumo-data-science/SKILL.md @@ -0,0 +1,330 @@ +--- +name: kumo-data-science +description: "Use for KumoAI graphs and predictive models on relational data in a warehouse, an object store or local files: inspect, visualize or export a graph's schema and relationships; train a model, score data with a trained one, explain its predictions, diagnose its training, publish, reload or retain it, train or score on custom labels or rows, or tune it. Covers churn, propensity, fraud, recommendation, forecasting and link prediction. For a new model it checks that the request names the data, outcome and time window, confirms the inferred tables and task, then generates one notebook or script; it runs it only when asked, in the notebook or as a host job." +--- + +# Data Science Agent — Router Skill + +## Build marker + +**`SKILL-BUILD: 2026-11-06-113fde01`** + +Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read +it from this file, you have not loaded the skill — say so rather than guessing. + +Targets the local `kumo.engine` API; the baseline bundle is `20261007+2e1eb12`. +Graph visualization requires a source build including [NVIDIA/kumo-engine#134](https://github.com/NVIDIA/kumo-engine/pull/134); it is absent from that bundle and the `1.0.0` release. +One SDK, no variants. + +## When to use + +Requests that mention any of: kumo, predict, churn, recommend, forecast, fraud, train a model, run +inference, explain a prediction, feature importance, feature attribution, column cohorts, model +metrics, trials, learning curves, publish or load a model, registry publishing, artifact retention, +caller-supplied labels, weights or splits, encoders, run mode, experiments, tune a model, semantic +view, metric view, ML Job, compute pool, Databricks job, graph diagram, visualize a graph, +inspect graph schema, export graph relationships. + +## Routing + +Some requests are not new-training intake. Route these to their own skill, which has its own +build and artifact checks: + +| Request | Skill | +|---|---| +| Show or export a graph; inspect its schema during model iteration | `kumo-graph-visualization/SKILL.md` | +| Attribution or column cohorts for an existing run | `kumo-model-explanations/SKILL.md` | +| Metrics, trial comparisons, learning curves, recorded progress | `kumo-training-diagnostics/SKILL.md` | +| Publish, reload, retain or retrieve a model or its artifacts | `kumo-model-persistence/SKILL.md` | +| Caller-supplied labels, weights, splits, or exact scoring rows | `kumo-custom-populations/SKILL.md` | +| Encoders, run mode, bounded experiments, model-plan options | `kumo-model-tuning/SKILL.md` | + +None of these require new-training intake, and none authorizes another fit. Preserve intent the +user has already given — task, target, run, model version, source, destination — and ask only +for what the selected workflow is missing. If the host has not provided the referenced skill, +report that limitation rather than inventing its workflow. **New training requests still follow +the gates below**, unchanged. + +A request to run the artifact as a **job** — a Snowflake ML Job on a compute pool, a Databricks +job on a cluster, unattended, or on a schedule — or a question about a job already submitted (status, logs, cost) also reads the +host's job file, when the host adapter names one. It adds settings to the gates below and changes +where Step 3 runs; it never replaces the gates. If the adapter names no job file, say that this +host does not run jobs and offer the notebook. A request that does not mention a job stays in the +notebook. + +## Purpose +Take a plain-language ML request, get it to a **confirmed** task and graph through a multi-turn +intake, and generate **one** runnable KumoAI artifact — a notebook for training, a Python script +for batch inference. The agent asks only for what is missing, infers structure rather than asking +for it, confirms before writing code, and does not run jobs unless told to. + +## What this skill does +1. **Accept** — a subDAG of checks; each node emits ACCEPT / ASK BACK / REJECT +2. **Preflight & graph gate** — infer tables, keys, time columns, links; user confirms +3. **Task gate** — restate entity / outcome / window / task type; user confirms +4. **Generate** — one notebook that **trains and then predicts** in the same process, with + exactly one blocking `fit` call; the predict call is part of the same artifact +5. **Run** — *only on explicit request* + +**The P1 deliverable is train → predict in one notebook.** Batch inference from a *saved* model +(`batch_inference`) is also supported: `Trainer.load` and `Trainer.load_from_connector` restore a +trainer, and `predict` runs against an explicitly rebuilt graph snapshot (see Step 2). + +## What this skill does NOT do +- Choose the predictive problem, target, horizon, graph, or tuning for the user — refuse and name + the minimum inputs needed +- Join, clean, deduplicate, aggregate, create labels, or transform data **as model input**. + Diagnostic scalar aggregates during preflight — reported as proposal evidence, never written and + never fed to training — are not this; they exist so a candidate key can be disqualified before + it silently discards rows +- Search for, rank, or curate tables by business relevance +- Retry or resume jobs, or keep state across sessions. The one execution the agent performs is + Step 3, and only when the user asks. +- Schedule anything on its own initiative (a recurring run is proposed and created only on explicit + confirmation, and only on a host whose job file covers it) +- RFM / zero-shot workflows +- Audit the user's schema for leakage at intake beyond the mandatory availability guards in + Node 1c. The codegen guard in `kumoai-reference/SKILL.md` (the target never appears as a + feature) still applies +- Turn a vague question into a complete request beyond the ask-back, or repair runtime job failures + +## Task scope +**Temporal binary classification**, **link prediction**, regression, ranking, static link +prediction, and embeddings as an output. Treat them all with the same intake; `time_window` is `not_critical` for +static shapes. + +--- + +## Step 1 — Accept subDAG + +Accept is a subDAG of check nodes run in a multi-turn chat. Each node produces one of: + +- **ACCEPT** — proceed to the next node. +- **ASK BACK** — ask one targeted question for exactly what is missing; re-run on the next turn. +- **REJECT** — cannot be fixed by clarification. Stop with a clear explanation. + +Ask **only** for missing, ambiguous, or contradictory inputs; never request secrets. +One ask-back turn lists every missing required field at once. + +### Node 1a — Operation and required fields +Reference: `accept/intake_contract.md` + +First decide the operation: `train` or `batch_inference`. + +**`train` — required (ask if missing):** +- `source_scope` — where the tables live, in a form the host accepts (`accept/intake_contract.md`) +- `prediction_entity` — what gets one prediction +- `prediction_target` — the outcome or interaction, with the positive-label condition +- `time_window` — `specified` / `not_critical` (static task) / `missing` → ask only when `missing`. + Static vs. temporal is decided by *what is predicted*, never by a date column in the data: a + value the entity row carries ("final price of each listing") or embeddings / observed links are + static; anything that happens or accumulates after a point in time is temporal + (`accept/intake_contract.md`, *static or temporal*). + +**`train` — never ask for:** column names, types, keys, joins, table mode (inferred at 1c); +output table (optional; preserve if given); metrics; embeddings (add only if asked). + +**`batch_inference`:** its required and resolved fields, and what changes at Nodes 1c, 1d and +Step 2, are in `accept/batch_inference.md`. Read it when the operation is `batch_inference`; a +training request never needs it. + +**Never fill a required field from the schema.** A preflight result (or a schema listing handed +to you) can tell you which tables exist, not what the user wants predicted. If the user has not +named the outcome, the positive-label condition, the item, the interaction, or the window, it is +missing — ASK BACK, even when the tables make a guess look obvious. + +**Quote test:** a required field is present only if you can quote the user's words for it; a +table or column name is not a quote. **Conversely, never ask the user which column or value +implements a field they already named** — infer the mapping and show it `[inferred]` at the gate +(`accept/intake_contract.md`). + +| Condition | Transition | +|---|---| +| Any required field missing | ASK BACK — list exactly the missing fields | +| `source_scope` given but does not resolve (location or semantic view not found) | ASK BACK — ask for a valid source; do not guess a similar name | +| All required present | proceed to Node 1b | + +### Node 1b — Atomic question check +Reference: `validate/atomic_question_check.md` + +| Condition | Transition | +|---|---| +| Goal is vague but potentially clarifiable | ASK BACK — ask the user to define the target concretely | +| Multiple goals bundled that cannot be split | REJECT — explain and ask for separate requests | +| "Figure out what to predict / pick the best target / tune and deploy" | REJECT — the agent does not choose the problem; list the minimum inputs for one supported request | +| Single, well-defined target | proceed to Node 1c | + +### Node 1c — Preflight & graph gate +Reference: `accept/preflight.md` + +First establish that you can read `source_scope` (`accept/preflight.md` §0). Then run read-only +preflight inside it. + +Present the graph proposal — seeds, tables, +primary keys, time columns, links, `table_mode: all-connected [default]` — each line tagged +`[supplied]` / `[declared]` / `[inferred]` / `[inferred, unverified]` / `[default]`. A semantic +view is resolved by `accept/semantic_view.md`, read only for a semantic view. If the user asked for embeddings, say they are included. **Stop and wait for confirmation.** +User-supplied values take precedence over inferred ones. + +**Mandatory column-availability pass.** Before presenting the graph, decide the selected columns +for every included table and add an `Excluded columns` section to the proposal. + +- When a table's `time_column` marks the beginning of a row's validity or an interval, exclude any + candidate feature whose value is determined only after that time: the interval's eventual + endpoint, its realized duration, or a summary of the completed interval. At the row timestamp + those values are future information. Decide by semantic role, never by matching column names. +- Separate the target column from columns derived from it; both rules apply at once. + - The target column itself **always stays** in the positional column list. The query references + it by name, so dropping it fails with a missing column. + - **Exclude** every other column whose value is a function of the target — a margin, ratio, + rate, share, rank or flag computed from it. Such a column reproduces the answer under a + different name, and the dependency is often visible only in the naming, so treat a plausible + derivation as a derivation and show it as an exclusion the user can overturn. +- Apply the active host adapter's runtime column constraints (`hosts//adapter.md`). A host + constraint may exclude a column; it must never invent a replacement semantic type. Exclude only + on evidence the adapter accepts as establishing the constrained property — a physical catalog + type generally does not establish a semantic one — or on a prior runtime-guard result. Where the + first pass cannot establish it, say so rather than guessing, and if the host's runtime guard + later resolves it, return to this node with a revised proposal rather than proceeding. + +Tag each exclusion `[inferred]`, give the reason, and let the user correct it at this same gate. +Carry the confirmed exclusion set into code generation: a table with any exclusion must be built +with an explicit positional column list, because omitting it silently restores every source +column. + +**Assert what preflight could not find.** The proposal must state these explicitly rather than +leaving them unmentioned, because each one changes whether the task is expressible: + +- a table with no **confirmed** primary key — say so. Until the user confirms one it is treated + as keyless: it may be an edge source, but not an edge destination and not the `FOR EACH` + entity. An `[inferred, unverified]` candidate does not lift that restriction; confirming it + does. Never promote a merely unique-looking column to primary key, and do not + conclude from a failed heuristic that the table must be an event or fact table. +- a time-ranged target whose referenced table has no usable time column — say so, and say what + would make the task expressible. A timestamp on another table does not repair the target. + +| Condition | Transition | +|---|---| +| You cannot read `source_scope` | ASK BACK — say what you checked; offer access or the structure in the user's own words (`accept/preflight.md` §0) | +| Seeds cannot be connected without transforming data | REJECT — explain; generate nothing | +| A semantic view fails a check in `accept/semantic_view.md` | ASK BACK — as that file says; do not pick a schema, drop tables, or substitute a link | +| User corrects a line | re-present the proposal with the correction `[supplied]` | +| Confirmed proposal still lacks metadata the task requires (entity key, target time column) | remain at Node 1c — explain the correction needed; generate nothing | +| User confirms | proceed to Node 1d | + +### Node 1d — Task interpretation gate +Restate, in this shape, then **stop and wait for explicit confirmation** — no code is +generated before the task is confirmed: + +``` +- Entity: one prediction per . +- Outcome: / +- Window: <0 to N days after the prediction anchor> | static +- Task type: +- Output: predictions written to in the same notebook | notebook only, no predictions written +- Scoring population (static tasks): rows whose is NULL | all rows, staged at +- Threshold (binary classification, predictions written): +- Embeddings: +``` + +**Static tasks: confirm the scoring population.** A static query scores only the rows whose +target is NULL (`kumoai-reference/references/kumo_finetuning_sdk.md` §7.1). If the user wants rows that +already have a target scored (for example "every order"), the notebook stages an explicit +population table. That is a write, and it goes in the source's system, so name it here: next to the confirmed +destination by default (`_POPULATION`) when the destination is in the source's +system; otherwise, or when there is no destination, ask for a schema or path in the source's +system. Never stage it in the source schema. + +| Condition | Transition | +|---|---| +| User confirms | ACCEPT — proceed to Step 2 | +| User changes intent | back to Node 1a with the new facts | + +**Do not emit any Kumo SDK code before this ACCEPT.** In the eval harness and in production alike, +a reply that reaches Node 1d ends with the restatement and the confirmation question. + +### Default metrics by task type +The generated notebook does not set metrics; the SDK's default model plan supplies them. Use +this table to tell the user what the model will be evaluated and tuned on. + +| Task type | Reported metrics | Tuned on | +|---|---|---| +| Binary classification | `auroc`, `ap`, `auprc` | `auroc` | +| Multiclass classification | `acc` | `acc` | +| Multilabel classification | `ap`, `auroc`, `auprc`, each `_macro` / `_micro` / `_per_label` | `ap_macro` | +| Regression, forecasting | `mae`, `mse`, `rmse` | `mae` | +| Ranking, link prediction | `map`, `precision`, `recall`, `f1`, each `@1` / `@10` / `@100` | `map@10` | +| Link prediction with a training-table weight column | `weighted_ndcg`, `weighted_recall`, `precision`, each `@1` / `@10` / `@100` | `weighted_ndcg@10` | + +Source: shipped package (`kumo_core.model_plan`) + +--- + +## Step 2 — Generate via KumoAI + +Read the host's `runtime.md` (`hosts//runtime.md`) before generating. + +After ACCEPT, load `kumoai-reference/SKILL.md` and hand off the confirmed payload. Produce +**one** artifact. + +### `train` → one notebook that fits, then predicts + +1. Connector from environment credentials; confirmed tables, metadata, links; graph and PQL + validation; SDK-default training table and model plan, no tuning. + After graph construction and validation, include the interactive graph cell from `kumo-graph-visualization/SKILL.md` for the user, before snapshot creation and training. + Visualization adds no approval gate; continue unless the user requests a pause for review. +2. **Exactly one** training call — `trainer.fit(...)`. It blocks until training finishes and + returns a `TrainingResult`; print its `metrics`. +3. **Predict in the same notebook** — same `Trainer`, same process, immediately after + `fit`: + - `output_location` confirmed at Node 1d → the notebook includes the `predict` call with a + `PredictionOutputConfig` pointing at that destination, written **verbatim**. + - no destination → predictions stay local; the notebook ends with a short markdown cell + describing the one line to add a destination later. That example is markdown text, never a + code cell: a notebook with no confirmed destination contains no code that writes. Inside + the markdown, show the code as text (a `text` fence), never as a `python` code block — + one that would run is a code cell that writes. + The predict call is part of the artifact, not a separate job the agent manages. The agent + still executes nothing (Step 3 is opt-in). + +The user may ask for a script instead of a notebook. +Include the SVG export from `kumo-graph-visualization/SKILL.md` before training, saving it alongside run outputs and reporting its path. + +### `batch_inference` + +See `accept/batch_inference.md`, *Step 2*. + +The artifact must preserve every confirmed input verbatim — entity, outcome, window, table mode, +metadata, links, destination. No invented link, horizon, or table. + +--- + +## Step 3 — Run **(only on explicit user request)** + +Default is generate-only. Run only when the user says to run it, +in the same or a later turn. Then, per `kumoai-reference/references/kumo_finetuning_sdk.md`: +- Run the notebook. `fit` blocks until training finishes; there is no Kumo job to poll. When the + user asked for a job, running means submitting it and waiting for its status as the host's job + file describes. +- Report the outcome: the training metrics, or the exception and the cell that raised it +- Let the notebook's own predict step run; predictions land at the confirmed destination — **the + name is written verbatim**, no suffix, no case change; report it in the destination's own + form (`DB.SCHEMA.TABLE`, or a path) +- Include embeddings if they were confirmed at Node 1d + +--- + +## Host adapter +This package carries exactly one host: the single directory under `hosts/`. Everything specific to +where you run and where the data lives is there — never assume another host's rules. + +- `hosts//adapter.md` — read at intake: how to check access to the source, the source and + destination forms the host accepts, how to run preflight, and runtime column constraints +- `hosts//runtime.md` — read at Step 2: how the generated code connects, authenticates, + writes and verifies + +--- + +## Extension points +- **Host expansion:** add a directory under `hosts/` with its own `adapter.md`. The portable core + (this file + `accept/` + `validate/` + `kumoai-reference/`) does not change. diff --git a/skills/kumo-data-science/accept/batch_inference.md b/skills/kumo-data-science/accept/batch_inference.md new file mode 100644 index 000000000..fb5374c88 --- /dev/null +++ b/skills/kumo-data-science/accept/batch_inference.md @@ -0,0 +1,191 @@ +# Batch inference — scoring with a saved model + +Read this file only when the operation is `batch_inference`: the user names a trained model and +wants it to score data. A training request never needs it. It adds to `SKILL.md` and +`intake_contract.md`; everything they say still applies unless this file overrides it. + +## Node 1a — fields + +**`batch_inference` — required (ask if missing):** +- `model_address` — the model the user names: a name or ID, a registry location with a version, or + a retained local bundle directory +- `output_location` — a destination the active host can write (`hosts//adapter.md`, + *Destinations*); a connector plus a table name is complete + +**`batch_inference` — resolved, never asked** (`accept/intake_contract.md`): +- `source_scope`: a source the user named, else the connector named for this request (only on a + host whose *Source scope* accepts a connector name), else the source in the team's training + record for the model (`kumo-model-persistence/references/reuse.md`) — the registry does not + keep one. The bundle restores trainer state, **not** the graph, so the graph is rebuilt from + source at 1c. The record is read at 1c, not at intake: a source that is not in the conversation + is not missing; ask for it only if 1c finds no training record. +- `scoring_population`: defaults to every entity of the trained entity table, anchored at the + latest timestamp in the rebuilt snapshot, tagged `[default]`. A subset or date the user names + replaces it. +- The target, positive-label condition, window and task type are fixed by the trained model and + must not be reopened. + +A request with a model and a destination goes to the gates. It does not get an ask-back. + +### `batch_inference` — required fields (ask if missing) + +Two fields are the user's to give. Everything else is reused from the model, resolved from what +the user already said, or defaulted — and shown at the gates for confirmation. + +| Field | What it must include | +|---|---| +| **`model_address`** | The model the user names, in their words: a model name or ID ("trainingjob-benchmark-001", "the benchmark model"), a registry location with a version, or a retained local bundle directory | +| **`output_location`** | Where scores are written, in a form the active host can write (`hosts//adapter.md`, *Destinations*). A connector plus a table name ("connector `benchmark_sf`, table `MODEL_SCORES`") is complete, because the connector supplies the database and schema. A fully qualified table (`DB.SCHEMA.TABLE`), or a path on a host that writes files, is also complete. | + +Both pass the quote test through earlier turns too. "Run the benchmark model again and put the +preds in the scores table", after a turn that named the model and the table, supplies both. + +### `batch_inference` — resolved, defaulted, or inferred (never ask) + +| Field | How it is obtained | Shown as | +|---|---|---| +| **task** — entity, outcome, window, task type | Reused from the model unchanged. Never reopened. | restated at 1d | +| **`source_scope`** | In order: a source the user named → the connector the user named for this request, on a host whose *Source scope* accepts a connector name (one connector reads the source and writes the scores unless the user separates them) → the source scope in the team's training record for the model (`kumo-model-persistence/references/reuse.md`); the registry keeps only `spec_version`, the query and `snapshot_id`, not the source. When neither of the first two applies, restate `Source: from the training record of [inferred]` and do not ask: the record is read at 1c. Ask only if 1c then finds no training record, or one that names no source. On a host that does not accept a connector name, a connector named alone is not a source: ask for the system and location as that host's adapter says. | `[supplied]` / `[inferred]` at 1c | +| **graph** | Rebuilt from the model's recorded graph definition inside `source_scope`, then confirmed | `[inferred]` at 1c | +| **`scoring_population`** | Default: every entity of the trained entity table, anchored at the latest timestamp in the rebuilt snapshot ("current predictions"). A static task has no anchor. If the user names a subset or an anchor date, use theirs. | `[default]` / `[supplied]` at 1d | +| **model version** | The version the user named. If the address resolves to more than one version and none was named, ask which one. Never pick the latest yourself. An address with a single version uses it. Either way, the generated code names the version explicitly. | `[supplied]` / `[inferred]` (the only version) at 1d | +| **embeddings** | Only when the user explicitly asks ("export the entity and target-item embeddings") | restated at 1d | +| **scoring options** | Set by the trained task type, below | restated at 1d | + +### `batch_inference` — scoring options by task type + +These are `predict` arguments, chosen at scoring time; none needs retraining, and the trained +query is never changed to get them (`kumoai-reference/references/kumo_finetuning_sdk.md` §9; +Source: shipped package (`kumo_core.prediction_options`)). The count is +`num_classes_to_return`; it is **independent of the query's `RANK TOP K`** (a `RANK TOP 10` +model can return 50), and when unstated it falls back to that K. + +| Task type | Count of results per entity | Threshold | +|---|---|---| +| Binary classification | none: the output already carries both class probabilities and the thresholded label, so a request for "both classes" needs no option | required, in [0, 1]: the user's value, else the one recorded with the model, else 0.5 `[default]` (`predict` has no default) | +| Temporal or static link prediction | **required** when predictions are written, 1 to 500; unstated, K from `RANK TOP K` `[default]`. Above 500, say so and offer up to 500 per entity, or the entity and item embeddings for a nearest-neighbour search over longer lists; ask which | none | +| Multilabel ranking | optional; unstated, K from `RANK TOP K`. No 500 cap | none | +| Multiclass, multilabel classification | optional; unstated, K from `RANK TOP K` when the query has one, else every class | multilabel only, in [0, 1]: the user's value, else 0.5 `[default]` | +| Regression, forecasting | none: one predicted value (or forecast series) per entity. A "top N" request does not apply; say so and ask whether that is what the user wants | none | + +A threshold outside [0, 1], or a count where the task takes none, is invalid: say which and ask. +A count or threshold the user asks for that fits these rules is carried as given, never turned +into a question and never a reason to retrain. + +**The saved bundle does not carry the graph.** It restores trainer state; `predict` takes an +explicit `snapshot`, so the source data must be available and the graph rebuilt and confirmed at +Node 1c exactly as for `train`. "Reuse the model" needs reading precisely here: the *task* is +reused unchanged and must not be reopened. The *graph* is rebuilt, but from the recorded definition, not re-asked. + +**When the request has a model and a destination, it goes to the gates. It does not get an +ask-back.** Do not ask for the source schema, the scoring population, the anchor date, or a fully +qualified output path when the rules above resolve them. Put each resolved value in the proposal, +tagged, so the user can correct it at confirmation. + +### `batch_inference` — invalid values (ask back, do not substitute) + +| Condition | Response | +|---|---| +| `model_address` does not resolve in the registry or as a bundle directory, or the user calls it rejected/invalid | Name it as unusable and ask for a corrected model identifier. Do not guess a similarly named model. | +| `output_location` is a kind the host cannot write (the adapter's *Destinations* lists what it can) | Name it as unusable here, say what the host writes instead, and ask for such a destination. Do not silently swap in another one. | + +When one field is invalid and the other is valid, ask only about the invalid one. When one is +missing and the other is present, ask only for the missing one. Do not also list the resolved +fields as questions. + +| Condition | Transition | +|---|---| +| `batch_inference`: `model_address` does not resolve, or the user calls it rejected | ASK BACK — name it as unusable, ask for a corrected identifier; do not guess | +| `batch_inference`: `output_location` is a kind the host cannot write | ASK BACK — name it as unusable, ask for a destination the host writes; do not substitute | + +## Node 1c — rebuilt graph + +For `batch_inference`, do not re-derive this set. The selected columns are the ones the model +was trained on: take them from the graph-building code and column metadata recorded with the +model (`kumo-model-persistence/references/reuse.md`), and show each table's list at this gate. +Re-inferring them can build the table with a different column list than the model saw, so +`predict` either fails on the mismatch or scores on different features. If no recorded column +selection is available, say so and remain at this node rather than substituting a fresh one. + +This node runs for `batch_inference` too. A saved bundle does not carry graph storage or source +data, and `predict` requires an explicit `snapshot`, so the graph is rebuilt and confirmed the +same way. Confirm that the rebuilt structure matches what the model was trained on; a bundle +from an older SDK, or source tables whose shape has drifted, fails at load or predict rather +than scoring silently. + +| Condition | Transition | +|---|---| +| `batch_inference` and no column selection is recorded with the model | remain at Node 1c — say the training-time selection is unavailable; generate nothing | + +## Node 1d — restatement + +For `batch_inference`, restate in this shape and stop for confirmation: + +``` +- Model: — trained task reused unchanged +- Source: [supplied | inferred] +- Population: as of latest snapshot timestamp [default] | the user's subset/date [supplied]> +- Output: predictions written to +- Scoring options: results per entity [supplied | default: RANK TOP K] (link prediction, ranking, multiclass, multilabel); none (regression, forecasting)> +- Embeddings: +``` + +Without tools, where 1c cannot run, fold the source and graph lines into this one restatement. +Do not turn them into questions. + +## Step 2 + +Follow `kumo-model-persistence/SKILL.md` for the load and predict code. Restore the trainer with +`Trainer.load` (retained bundle) or `Trainer.load_from_connector` (registry), materialize the +snapshot for the graph confirmed at 1c, build the prediction table for the confirmed scoring +population — `kumo-custom-populations/SKILL.md` when the user supplied exact rows — and call +`predict(prediction_table, snapshot=snapshot, output_config=...)`. + +Do not refit. If the bundle cannot be loaded, or the rebuilt graph is incompatible with it, +report the failing stage and stop; do not substitute a fresh training run without being asked. + +## Examples + +### Model and destination given → gate + +> "Load trained model `trainingjob-benchmark-001` and write current batch predictions through +> connector `benchmark_sf` to table `MODEL_SCORES`." + +Node 1a: `model_address` ✓ and `output_location` ✓. A connector plus a table name is a complete +destination. Nothing is asked. The rest is resolved and goes straight to the gate: + +``` +- Model: trainingjob-benchmark-001 — trained task reused unchanged +- Source: benchmark_sf [inferred — the connector named for this request] +- Population: all entities of the trained entity table, as of the latest snapshot timestamp [default] +- Output: predictions written to MODEL_SCORES via benchmark_sf +Shall I generate the batch-inference notebook with this setup? +``` + +With "…export the entity and target-item embeddings" added, the gate also carries +`Embeddings: included`. It is still one confirmation, not an ask-back. + +### Destination missing → ASK BACK + +> "Use trained model `trainingjob-abc123` to score current accounts." + +- `model_address` ✓ · `output_location` ✗ +- Not asked: the source (from the model's training record), the population ("current accounts" means all + accounts at the latest snapshot timestamp, the default), and the task. The task is never + reopened. + +> "Where should the scores be written? For example a connector plus a table name, or +> `DB.SCHEMA.TABLE`." + +### Destination unusable → ASK BACK + +On a host whose *Destinations* are Snowflake tables only: + +> "Score with `trainingjob-abc123` and write predictions to `s3://tmp/scores`." + +- `model_address` ✓ · `output_location` invalid: not a Snowflake table + +> "`s3://tmp/scores` isn't a destination I can write to here, because scores go to a Snowflake +> table. Which Snowflake table should receive them?" + +Only the invalid field is raised. Do not add questions about the source or the population. diff --git a/skills/kumo-data-science/accept/intake_contract.md b/skills/kumo-data-science/accept/intake_contract.md new file mode 100644 index 000000000..a2ffe2f7e --- /dev/null +++ b/skills/kumo-data-science/accept/intake_contract.md @@ -0,0 +1,177 @@ +# Input Contract — What a request must contain + +The contract is keyed by **operation**. Decide the operation first, then apply that operation's +table. The tables below are the complete contract: a field that is not in them is not required. + +Three kinds of field exist, and the distinction is the whole point of this file: + +| Kind | Who supplies it | If absent | +|---|---|---| +| **Required** | the user | ASK BACK — ask for exactly these, nothing else | +| **Inferred, then confirmed** | the agent, by read-only preflight (`accept/preflight.md`) | never asked; shown as `[inferred]` and confirmed at the graph gate — except a key that preflight could only fail to disqualify, which is shown `[inferred, unverified]` and cannot be used until confirmed | +| **Optional** | the user, if they want | never asked; defaults apply; if supplied it must be preserved verbatim | + +**Infer structure, never infer intent.** Column names, keys, time columns and links are structure +— measure them. Entity, outcome, positive-label condition, interaction, item, window and +destination are intent — the user's, and only theirs. No outcome definition is ever invented. + +A preflight result or schema listing **never fills a required field.** Seeing a `sales_events` +table does not tell you the positive interaction is "a sale"; seeing a `target_label` column does +not tell you the positive-label condition. If the user did not state it, it is missing — ask, +even when the schema makes the answer look obvious. Preflight may resolve *which table holds* +what the user named; it may not supply what they did not name. + +**The quote test — is a required field present?** A required field is present only if you can +quote the user's own words for it, from any turn, in any vocabulary: "each customer", "a +cancelled booking", "at least one fraud event", "next 30 days". A table name, a column name, or "the only +entity in the schema" is not a quote — the field is missing, and the reply is an ask-back, not a +graph proposal. + +**Named intent, inferred mapping — never ask the user to map their words onto columns.** Once the +user has named a field, turning their words into a physical table, column, or value is structure. +Preflight infers it — "a cancelled booking" → `bookings` where `status` marks a cancellation — +and it is shown as `[inferred]` in the proposal for the user to confirm or correct. Do +not ask which column or value implements something the user already named, and do not ask them to +re-define it more precisely. Inferred values are fine as long as the complete proposal is shown +and confirmed; questions are only for missing, ambiguous, or contradictory inputs. + +The two rules are one rule seen from both sides: the **user's words** decide *whether* a field is +present; the **data** decides *how* a present field is implemented. + +--- + +## Operation + +`train` or `batch_inference`. Read it from the request: "train / build a model / notebook that +predicts" is `train`; "use my trained model / score / run inference with job X" is +`batch_inference`. One session covers one operation, one task, one graph, one artifact. +Ask only if the request genuinely reads both ways. + +--- + +## `train` — required fields (ask if missing) + +| Field | What it must include | +|---|---| +| **`source_scope`** | Where the tables live, in a form the active host accepts (`hosts//adapter.md`, *Source scope*): for example a database + schema, a dataset, a catalog + schema, a bucket prefix or a directory of files, or a connector name the host knows. On a host that supports them, a fully qualified semantic view (`DB.SCHEMA.VIEW`) contributes exactly its base tables, which must share one schema (`semantic_view.md`). This is where tables are enumerated. Credentials are never requested in chat. | +| **`prediction_entity`** | What receives one prediction — "each account", "each customer". A physical table/key is welcome but not required; preflight resolves it. | +| **`prediction_target`** | The observable outcome (classification/regression) or the positive interaction (link prediction), **stated by the user**. For classification this includes the positive-label condition — "at least one fraud event"; "predict fraud" alone leaves it missing. For link prediction and ranking it includes the **item** being recommended and the **interaction** that counts as positive — "spare-part families the customer ordered"; "recommend to customers" alone leaves both missing. | +| **`time_window`** | Tri-state. `specified` ("next 30 days") → pass. `not_critical` → the task is static (no temporal component: static link prediction, static regression/classification) → pass, do not ask. `missing` → the task is temporal and no window was given → ASK BACK. Decide static vs. temporal with the test below. | + +**Static or temporal — decide from what is being predicted, never from the data.** Almost every +table has a date column (`booked_date`, `event_at`); its presence says nothing about the task. + +- **Static** (`not_critical`, do not ask): + - the target is a value the entity row itself carries — "the final price of each listing", + "the category of each article". The entity is one record; nothing has to happen after it. + - the user asks for **embeddings**, or for links **learned from observed activity** without a + forecasting framing — "learn user and product embeddings from past clicks", + "suggest tags for each article from the article-tag links we already have". +- **Temporal** (window required): the target is something that happens, or is aggregated, over a + period after a point in time — "will churn", "will have a fraud event", "total spend", + "what each customer will reorder", and recommenders, which by default predict future + interactions ("a recommender for each member"). + +If the task is static, restate `Window: static` at the task gate and do not ask. A time column +may still be used as the graph's time column — that is structure, not a window. + +A request that names the dataset ("the e-commerce fraud dataset", a schema name) has supplied +`source_scope` if that name resolves to a location the host accepts. Do not ask the user to restate a +location they have already given in an earlier turn. + +## `train` — inferred, then confirmed (never ask) + +| Field | How it is obtained | +|---|---| +| **seed tables** | The physical tables behind the entity and the target, found by preflight inside `source_scope` | +| **primary keys, time columns** | Preflight; each shown as `[inferred]` unless the user stated it (`[supplied]`) or a semantic view declares it (`[declared]`). A key that preflight could only fail to disqualify is shown `[inferred, unverified]` and must be confirmed before the table can be an entity or an edge destination | +| **links** | Preflight; shown as `[inferred]`, or `[declared]` when read from a semantic view | +| **`table_mode`** | Default `all-connected` — every table reachable from the seeds. Shown as `[default]` in the graph proposal; the user may switch to `named-table` there. | + +User-supplied values always win over inferred ones. +The complete proposal — every table, key, time column, link, and the table mode — is shown and +confirmed **before** any PQL is written. + +## `train` — optional (never ask; preserve if given) + +| Field | Default | +|---|---| +| **`output_location`** | none — training yields a notebook; when given, the same notebook also predicts into it (SKILL.md Step 2) | +| **artifact format** | one notebook (user may ask for a script) | +| **embeddings** | not included; add only when the user explicitly asks (see below) | +| **binary classification threshold** | `0.5`, shown `[default]` at Node 1d whenever a binary classification writes predictions. `predict` requires one and has no default of its own (`kumoai-reference/references/kumo_finetuning_sdk.md` §9) | +| **metrics** | by task type, see `SKILL.md` | + +If the user supplies an `output_location`, carry it verbatim into the task interpretation gate +and the generated artifact — the notebook's predict call writes there. If none is supplied, +predictions stay local and the notebook shows how to add a destination. The destination must be +one the host can write (`hosts//adapter.md`, *Destinations*). **A table is written with +exactly the name passed** — no suffix, no case change (the host's `runtime.md`, +`kumoai-reference/references/kumo_finetuning_sdk.md` §9.1). A file path follows §9.2. Older SDK builds appended +`_predictions`; the current one does not. + +--- + +## `batch_inference` + +Required, resolved and invalid fields for scoring with a saved model are in +`batch_inference.md`. Read it only when the operation is `batch_inference`. + +--- + +## Embeddings + +Default to predictions alone. Add embeddings **only when the user explicitly asks for them** — +never ask back for this, and never infer it from the task being recommendation-shaped. + +Recognise: "we need shopper and item embeddings", "learn customer and product embeddings from +completed orders", "export the entity and target-item embeddings from that model". + +When embeddings are requested, say so explicitly in every reply that restates the request — the +graph proposal and the task interpretation gate — the user must see that embeddings are part of +the output. Carry it into codegen as +`PredictionOutputConfig(artifact_type=...)`; see `kumoai-reference/references/kumo_finetuning_sdk.md` §9. + +--- + +## Canonical good example (`train`) + +> "Create a notebook that trains a Kumo model for our fraud use case. Data is in +> `USER_DATA.E_COMMERCE`. Entity: each customer. Outcome: at least one fraud event. Window: next 30 days." + +What makes it good: +- `source_scope`: `USER_DATA.E_COMMERCE` ✓ +- `prediction_entity`: each customer ✓ +- `prediction_target` with positive-label condition: at least one fraud event ✓ +- `time_window`: `specified`, 30 days ✓ +- No columns, keys or joins described — **correct**; preflight infers them and the user confirms. + +## Canonical bad example (`train`) + +> "Use my sales data to figure out which customers are risky." + +What is missing: +- `source_scope`: "sales data" does not resolve to a location ✗ +- `prediction_target`: "risky" is not an observable outcome ✗ +- `time_window`: `missing` for what reads as a temporal question ✗ + +Not missing, and not to be asked for: schema, join keys, output table. + +--- + +## What to do when a required field is missing + +Ask for **exactly** the missing required fields, once, in one turn. Do not ask for structure, +table mode, or an output table on a training request. On a `batch_inference` request, do not ask +for the source schema, population, or anchor date (see the *resolved, defaulted, or inferred* +table). Do not invent values. Ask in the user's +vocabulary ("what should be recommended to each customer?"), and do not attach questions +about columns or values for the fields that *are* present — those are confirmed later, as +`[inferred]`. Example: + +> "To proceed I need: +> 1. **Where the data lives** — for example a database and schema, or a folder of files. +> 2. **The prediction window** — e.g. 'in the next 30 days'." + +Ask only for missing, ambiguous, or contradictory inputs, restate the resolved contract, and never +request secrets in chat. diff --git a/skills/kumo-data-science/accept/preflight.md b/skills/kumo-data-science/accept/preflight.md new file mode 100644 index 000000000..5c6447227 --- /dev/null +++ b/skills/kumo-data-science/accept/preflight.md @@ -0,0 +1,225 @@ +# Read-only Preflight — inferring structure before the graph gate + +Once the required fields are present, the agent resolves **structure** itself. It does not ask +the user for column names, keys, or joins, and it builds no Kumo table or graph at this point — +preflight is a catalog read of the source plus explicitly targeted, read-only probes against +source data, and it is read-only throughout. A probe exists only to produce evidence for the +proposal; it is never written anywhere and never becomes a model input. + +**What** preflight establishes is portable and written here. **How** it reads the source — which +tool, which catalog query — belongs to the active host, in the *Access* and *Preflight* sections +of `hosts//adapter.md`. The examples below are SQL; a host whose source has no SQL answers +the same questions with its own reader. + +Preflight produces the **graph proposal** the user confirms at Node 1c. Every value carries a tag: + +| Tag | Meaning | +|---|---| +| `[supplied]` | the user stated it; never overridden | +| `[declared]` | read from a semantic view's definition (`semantic_view.md`); shown for confirmation | +| `[inferred]` | measured by preflight; shown for confirmation | +| `[inferred, unverified]` | a candidate preflight could not establish, only fail to disqualify; must be confirmed before use | +| `[default]` | a contract default (`table_mode: all-connected`); shown for confirmation | + +If the harness or host has already run preflight and handed you a result (for example a +"Preflight result for " message), use that result as-is — do not ask the user to repeat it. + +--- + +## 0. Access — can you read the source? + +Before §1, establish whether **you**, in this session, can read `source_scope` with a read-only +tool you already have. The host adapter's *Access* section says which tools count and the check +that proves each one reaches the data. + +- **You can read it** → run §1-§6 through that tool, and name the tool in the proposal's + `Source scope` line. +- **You cannot** — no suitable tool, or the tool reaches the system but not this scope (an + authorization error, an unknown schema or bucket) → stop before §1. In one message, tell the + user exactly what you checked and what happened, quoting any error, and offer both ways forward: + 1. **Access** — they make a read tool available, as the adapter describes, and tell you; you + then repeat the check. + 2. **Details** — they paste the structure: tables, columns with types, and any declared keys, + in whatever form they have (the adapter lists useful ones). You build the proposal from it. + + This is one ask-back. Do not also ask for anything §1-§8 would infer. +- **Never** ask for a credential's value or invite one into the chat, sign in, change a tool's + configuration, or set up a tool on the user's behalf. Never broaden the search beyond + `source_scope` because the scope itself was unreadable. + +A proposal built from the user's details follows §1-§8 unchanged, except that nothing that needs +row data can run: the §3 rule 5 probe is skipped, so its candidates stay `[inferred, unverified]`. +Say in the `Source scope` line that the structure came from the user, not from a read. + +This check concerns your session only. The generated artifact authenticates at run time through +its own connect step (the host's `runtime.md`), so your access — or its absence — changes +neither the generated code nor any gate. + +--- + +## 1. Resolve `source_scope` + +If `source_scope` is a semantic view (`DB.SCHEMA.VIEW`), read `accept/semantic_view.md` and +resolve it there instead of here; then continue at §2. + +```sql +SELECT SCHEMA_NAME FROM .INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME ILIKE ''; +SELECT TABLE_NAME, ROW_COUNT FROM .INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA ILIKE ''; +``` + +- Not found, or zero tables → **ASK BACK**: the source cannot be resolved; ask for a valid + location. Do not guess a similarly named schema, dataset, bucket or directory. +- Found → enumerate tables **only inside this scope**. Never search other schemas, and never + rank or curate tables by business relevance. + +## 2. Columns and types + +```sql +SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE +FROM .INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA ILIKE '' +ORDER BY TABLE_NAME, ORDINAL_POSITION; +``` + +## 3. Primary key per table + +In order of preference, first match wins: +1. A declared primary key, where the source declares any (the adapter's preflight shows how to + read them; files and object stores declare none). +2. `_ID`, where the stem is the table name without a `_DIM` / `_PROFILE` / `_EVENTS` + suffix, singular or plural (`CUSTOMER_ID` in `CUSTOMERS`, `PAGE_VIEW_ID` in `PAGE_VIEW_EVENTS`). +3. A column named `ROW_ID`, `ID`, or `EVENT_ID`. +4. For dimension / profile tables only: `_ID` for any token of the table name + (`ACCOUNT_ID` in `ACCOUNT_PROFILE`). Never for event tables — there such a column is a link. +5. A column that one or more *other* tables in scope carry under the same name is a **candidate** + only. Shared naming does not make it a key, and neither does uniqueness: `COUNTRY`, `STATUS` or + `MERCHANT` can be ordinary attributes, and an accidentally unique attribute is still not an + identifier. A duplicated inferred key silently discards rows. A column that rules 1-4 already + established as another in-scope table's key is a link to that table, not a candidate here — + skip it, as rule 4 does for event tables. + + Use a targeted read-only probe to **reject** candidates, never to establish identity: + + ```sql + SELECT COUNT(*) AS N, COUNT(DISTINCT ) AS N_DISTINCT, COUNT() AS N_NON_NULL + FROM ..
; + ``` + + Anything other than `N > 0 AND N_DISTINCT = N_NON_NULL = N` disqualifies the candidate outright + — an empty table returns three zeros and does not qualify. Surviving the probe makes it a + **proposed** key, tagged `[inferred, unverified]`, which the user confirms or corrects at the + gate before it is used as the `FOR EACH` entity or an edge destination. If several candidates + survive on one table, or the same candidate survives on more than one side, report the + ambiguity and propose none. If the probe cannot run, list the column under `Not found` as an + unverified candidate. + +6. Otherwise preflight **inferred no primary key**. Report that as its own line and treat the + table as keyless unless corrected. A keyless table may be an edge *source*, but not an + `Edge.dst_table` and not the `FOR EACH` entity. Do not conclude from this failure alone that + the table is an event or fact table — it may have a natural, composite or unconventional key + the heuristic missed. Never promote a merely unique-looking column to primary key: a column + unique in the rows you can see is not a declared key, and a child table may hold several rows + per parent. + +Rules 1-4 establish a key and name it `[inferred]`. Rule 5 never establishes one: it proposes a +survivor as `[inferred, unverified]`. Until the user confirms it, the table is treated exactly as +keyless — usable as an edge source, but not as an edge destination and not as the `FOR EACH` +entity. Confirmation is what lifts that restriction, not the existence of a candidate. + +## 4. Time column per table + +The first `TIMESTAMP*`, `DATE`, or `DATETIME` column, preferring names ending in `_AT`, `_TS`, +`_TIME`, `_DATE`, or named `EVENT_TIME` / `TIMESTAMP`. A table with none is static. + +For every time-ranged target aggregation, the table it references must have a usable +`time_column`. If it does not, say so in the proposal and say what would make the task +expressible: a timestamp on some other table does not repair the target. + +A catalog `TEXT` column may still be usable when its date semantics are established and the +generated code casts it with `dtype=Dtype.date`; the absence of a timestamp-typed catalog column +is not by itself grounds for rejection. Integers that merely order or describe events — a +sequence number, a day-of-week, an hour, a gap in days since a previous event — are not +timestamps and must not be treated as one. + +## 5. Column availability + +For each table, place every candidate feature in time relative to that table's proposed +`time_column`, then exclude the ones that are not yet knowable at it. + +- When the time column marks the beginning of a validity period or interval, exclude a feature + whose value is only determined once that interval later changes or ends: its eventual endpoint, + its realized duration, or any summary of the completed interval. +- Exclude a feature computed from, or containing, the confirmed prediction target. +- Apply the active host adapter's runtime column constraints. + +Decide by semantic role, never by a fixed column-name list. Record every exclusion and its reason +in the proposal. Where the role is genuinely ambiguous, do not invent one — show the column as an +exclusion the user can overturn at the gate, rather than asking the user for column names. + +## 6. Links + +A link is proposed when a column in table A has the **same name** as the primary key of table B +(`FRAUD_EVENTS.CUSTOMER_ID → CUSTOMERS.CUSTOMER_ID`). Foreign-key constraints, if declared, are +taken as well, where the source declares any. Nothing else is inferred — no fuzzy matching, +no value overlap checks, no joins executed. + +## 7. Seeds and table mode + +- **Seeds** are the tables behind `prediction_entity` and `prediction_target` (for link + prediction: source entity, item entity, and the interaction table). Resolve them by name + against the enumerated tables; if none matches, ask the user which table holds the entity or + the outcome — that is intent, not structure. +- **Preflight runs only after Node 1a passes.** It resolves tables for fields the user already + supplied; it never back-fills a field the user left out. If the required-fields check found + `positive_interaction`, `recommended_item`, `positive_label_condition` or `time_window` + missing, ask for them first — do not read them off the schema. +- **`table_mode: all-connected [default]`** — every table reachable from the seeds through the + proposed links is included. List any table that is **not** reachable so the user can drop it + or name a link. +- If the seeds themselves cannot be connected through inferred links, say so and **generate + nothing** — connecting them would require creating or transforming data. + +## 8. The graph proposal + +Present exactly this shape, then **stop and wait** for confirmation: + +``` +Source scope: USER_DATA.E_COMMERCE [supplied] — 7 tables found, read via SQL +Table mode: all-connected [default] — 7 of 7 tables reachable from the seeds + +Metadata +- customers: primary key CUSTOMER_ID [inferred] +- fraud_events: primary key EVENT_ID [inferred] +- fraud_events: time column EVENT_AT [inferred] +- ... + +Links +- fraud_events.CUSTOMER_ID -> customers.CUSTOMER_ID [inferred] +- ... + +Excluded columns +- fraud_events.RESOLVED_AT [inferred] — not knowable at the row's EVENT_AT +- fraud_events.DAYS_TO_RESOLVE [inferred] — duration of an interval that has not ended yet +- ... + +Not found +- payments: no primary key inferred [inferred] — treated as keyless; edge source only unless corrected +- events: no usable time column [inferred] — a time-ranged target cannot reference this table +- ... + +Confirm, or correct any line (your values take precedence). +``` + +`Excluded columns` and `Not found` are local availability-gate extensions to the confirmation +format. + +## 9. Never + +- Prepare, transform, aggregate or materialize **model inputs** — joining, cleaning, + deduplicating, creating labels. A diagnostic scalar aggregate over source rows is permitted + solely as evidence for the proposal: row, distinct and null counts for a candidate key. Its + result is reported and then discarded; it is never written anywhere and never reaches training. +- Rank tables by relevance or drop tables the user did not ask to drop. +- Proceed to PQL before the proposal is confirmed. +- Ask the user for columns, types, keys or joins — with one exception: when §0 finds you cannot + read the source, offering to take its structure from the user is the §0 ask-back. Whatever the + proposal is built from, it is still tagged, still confirmed, and contains no invented values. diff --git a/skills/kumo-data-science/accept/semantic_view.md b/skills/kumo-data-science/accept/semantic_view.md new file mode 100644 index 000000000..c77af159c --- /dev/null +++ b/skills/kumo-data-science/accept/semantic_view.md @@ -0,0 +1,146 @@ +# Semantic view as `source_scope` + +Read this file only when `source_scope` is a fully qualified semantic view (`DB.SCHEMA.VIEW`). It +replaces `preflight.md` §1 for that case; every other preflight section still applies. Nothing here +applies to a schema or connector. + +## Resolve the view + +A semantic view is not a schema to search. It is a definition that points at physical tables and +declares their keys and relationships. Read the definition; do not query the view itself. + +```sql +DESCRIBE SEMANTIC VIEW ..; +``` + +The result holds one `property` / `property_value` pair per object, not one column per field. +Group the rows by `object_name` for each `TABLE` and `RELATIONSHIP` object, then read each field +as "the row whose `property` is X, taking `property_value`": for a `TABLE` object read +`BASE_TABLE_DATABASE_NAME`, `BASE_TABLE_SCHEMA_NAME`, `BASE_TABLE_NAME` and `PRIMARY_KEY` this way; +for a `RELATIONSHIP` object read `TABLE`, `REF_TABLE`, `FOREIGN_KEY` and `REF_KEY` this way — +`TABLE` and `REF_TABLE` are the view's logical names too; map each to its physical table through +the extracted `TABLE` records before using it anywhere downstream. + +- Not found or unreadable → **ASK BACK**, exactly as for a schema that does not resolve. +- **A logical table with no `BASE_TABLE_*` properties does not name a physical table.** One + defined by a query carries its own definition instead. The SDK can read a query through + `connector.sql(...)`, but this skill does not build graph tables from SQL it composed itself, so + the view alone is not enough. **ASK BACK**: name the logical table, say it is defined by a query + rather than a base table, and ask for a view whose tables are physical or for the physical table + to use. Do not parse the definition or rebuild it as a query of your own. +- **Only a plain column-to-column equality relationship becomes an edge.** A Kumo `Edge` is + foreign key to primary key, matched on equality. If a relationship declares any other kind of + match, do not read its `FOREIGN_KEY` / `REF_KEY` as an ordinary key pair: a non-equality payload + can read as a one-column key and become an edge that does not mean what the view says. + **ASK BACK**: name the relationship and its kind, and say only equality joins become edges. +- **The base tables must all sit in one database and schema.** The connector reads from a single + schema. If they span more than one, **ASK BACK**: name the schemas found and ask for a semantic + view or a schema whose tables share one schema. Do not pick one schema, and do not drop tables + to make it fit. +- That one schema becomes the effective scope: it is the `database` / `schema_name` passed to + the connector, and `preflight.md` §2 reads columns from it. +- **The tables in scope are exactly the view's base tables** — not every table in that schema. + The user chose them by naming the view; that is their selection, not curation by the agent. +- **Always use physical names.** The view's logical names (`USERS`, `TRANSACTIONS`) do not exist + for Kumo. Every table, key, link, edge and PQL reference uses the base table and column names, + with the same case and quoting the catalog reports. +- Each `PRIMARY_KEY` and each relationship is `[declared]`. They replace `preflight.md` §3 and §5 + for the tables they cover; those sections still apply to any base table the view gives no key + or link for. +- A key with more than one column, on either side of a relationship, cannot become a Kumo edge. + **ASK BACK**: name the relationship and say a single-column key is needed, the same way as a + view that does not resolve — ask for a corrected view or schema. Do not offer to drop the + table, concatenate the columns, or substitute a different link as options. Those are repairs, + not a corrected source, and a menu of repairs is not an ask-back. +- A declared `PRIMARY_KEY` with more than one column only matters when it is what some + relationship's `REF_KEY` points **at**. If no relationship's `REF_KEY` is this composite key, it + needs no ask-back by itself — not even when this table is the **foreign-key** side of its own + relationships and one of those `FOREIGN_KEY` columns happens to also be part of the composite + key; pointing out of a table never depends on that table's own primary key. Leave the table in + the graph exactly as `preflight.md` §3 treats a table with no usable primary key: keyless, or + with a single-column `[inferred]` key (e.g. an `*_ID` column) offered for confirmation. That + keyless or `[inferred]` treatment applies only while **no** relationship references this table + at all. If some relationship's `REF_KEY` is a different single column of it, that column is the + key, by the bullet below; do not infer another, because an edge points at the key Kumo is given + and a guessed one joins the foreign key to the wrong column. Ask back only when some + relationship's `REF_KEY` is the composite key — that is the previous bullet, not this one. +- A relationship whose **single-column** `REF_KEY` is not the referenced table's declared + `PRIMARY_KEY` (for example it references a `UNIQUE` key instead) is still expressible. A Kumo + primary key is declared by the caller in `Table.from_source_table(..., primary_key=)` and does + not have to match the catalog, so that column becomes the table's Kumo primary key, shown + `[declared]`. Say in the proposal that it differs from the catalog's primary key so the user can + correct it. This is not a repair; it is reading the view as written. +- That holds only while one column can serve the whole table. When **two relationships reference + two different columns of the same table**, a Kumo table has one primary key and cannot carry + both. **ASK BACK**: name both relationships and the column each references, say only one can be + an edge destination, and ask for a corrected view, or for explicit authorization to build without + one named relationship. Asking only which column to key on is not enough, because either answer + still leaves the other relationship unbuildable. Do not silently key on one and leave the + other relationship unbuilt, and do not emit the table twice to satisfy both: separate copies are + not joined to each other, so `A -> Entity <- B` would silently become two disconnected + components, and an entity split this way is ambiguous as the `FOR EACH` target. +- Ignore the view's dimensions, facts and metrics. They describe aggregate reporting, not the + graph. A metric is not the user's outcome: if the user has not described one, `prediction_target` + is missing, as for any source. If they have ("completed orders", "fraud events"), map their + words to columns as for any source and show the mapping `[inferred]` at the gate — do not ask + which column or value implements it. + +## Tag precedence + +Precedence: `[supplied]` > `[declared]` > `[inferred]`. `[inferred, unverified]` is not a rung +on that ladder but a weaker form of `[inferred]`: whatever its source, a value carrying it must +be confirmed before the table can be an entity or an edge destination. + +## In the graph proposal + +When `source_scope` is a semantic view, the first line names the view and the schema it resolved +to, and keys and links read from it carry `[declared]`: + +``` +Source scope: semantic view SALES_DB.ML.CHURN_SV [supplied] — 3 base tables in SALES_DB.ML +... +- orders: primary key order_id [declared] +- orders.customer_id -> customer_profile.customer_id [declared] +``` + +## Examples + +### Valid view → graph proposal + +> "Train a churn model on the semantic view `SALES_DB.ML.CHURN_SV`. Entity: each customer. +> Outcome: no order in the next 60 days." + +Node 1a: `source_scope` is the semantic view ✓ — nothing else is asked. At Node 1c the agent runs +`DESCRIBE SEMANTIC VIEW`, finds three base tables, all in `SALES_DB.ML`, and replies — then stops: + +> Source scope: semantic view `SALES_DB.ML.CHURN_SV` [supplied] — 3 base tables in `SALES_DB.ML` +> Table mode: all-connected [default] — 3 of 3 tables reachable from the seeds +> +> **Metadata** +> - customer_profile: primary key customer_id [declared] +> - orders: primary key order_id [declared]; time column ordered_at [inferred] +> - support_tickets: primary key ticket_id [declared]; time column opened_at [inferred] +> +> **Links** +> - orders.customer_id -> customer_profile.customer_id [declared] +> - support_tickets.customer_id -> customer_profile.customer_id [declared] +> +> Confirm, or correct any line — your values take precedence. + +The proposal uses the physical names (`customer_profile`, not the view's logical `CUSTOMERS`), +lists only the view's three tables even if `SALES_DB.ML` holds more, and the connector is built on +`SALES_DB` / `ML`. The view's metrics play no part — the outcome is still the user's words. + +### Base tables span more than one schema → ASK BACK + +> "Predict 30-day churn for each customer using the semantic view `SALES_DB.ML.CHURN_SV`. +> Churn means no order in the window." + +Preflight: `DESCRIBE SEMANTIC VIEW` finds base tables in two schemas — `customer_profile` in +`SALES_DB.CRM`, `orders` in `SALES_DB.ML`. + +> "`CHURN_SV` reads from two schemas — `SALES_DB.CRM` (customer_profile) and `SALES_DB.ML` +> (orders). I can only build a graph from tables in a single schema. Could you point me to a +> semantic view or a schema where these tables sit together?" + +Not done: picking one schema, dropping a table, or copying data across schemas. diff --git a/skills/kumo-data-science/hosts/generic/adapter.md b/skills/kumo-data-science/hosts/generic/adapter.md new file mode 100644 index 000000000..f907eb1ee --- /dev/null +++ b/skills/kumo-data-science/hosts/generic/adapter.md @@ -0,0 +1,149 @@ +# Host Adapter — Generic coding agent + +For a coding agent working beside the user — Claude Code, Codex, or any agent with a shell and +file tools — rather than inside a data platform. The agent produces code that runs in **any** +Python environment where Kumo is present, against whichever of the six Kumo connectors the data +calls for. The agent does not know where that code will run; the user does. + +Kumo is already present in the environment that runs the artifact; never raise environment setup +with the user. Generating the artifact — connect step, credentials, destinations and read-back — +is in `runtime.md` in this directory; read it at Step 2. + +This host has no job file: it does not submit, schedule or monitor jobs. When asked for one, say +so and offer the notebook or script. + +--- + +## 1. Source scope + +`source_scope` names a system and a location in it. Map it to its connector +(`kumoai-reference/references/kumo_finetuning_sdk.md` §3, §3.4): + +| The user names | Connector | +|---|---| +| A local directory of CSV or Parquet files | `FileConnector` | +| `s3://bucket/prefix` | `S3Connector` | +| `gs://bucket/prefix` | `GCSConnector` | +| A BigQuery project and dataset | `BigQueryConnector` | +| A Databricks catalog and schema | `DatabricksConnector` | +| A Snowflake database and schema | `SnowflakeConnector` | + +- A dotted name with no system named (`SALES.EVENTS`) fits Snowflake, BigQuery and Databricks + alike. Ask which system holds it — that one question, nothing else. A system already named in + any turn, or evident from the user's tools or files, settles it. +- A connector name on its own is not a location on this host, which keeps no registry of named + connectors. Ask what system and location it points at. +- A Snowflake semantic view resolves through `kumo-data-science/accept/semantic_view.md`, which + needs Snowflake SQL (§2). Without it, the §2 details are the view's base tables, their keys and + its relationships. +- Values only the runtime needs — a Snowflake warehouse, a Databricks SQL warehouse or cluster, a + `connections.toml` entry — are not part of `source_scope` and are never asked for at intake. + `runtime.md` §1 handles them. + +ADLS has no Kumo connector. Data there is outside this host's reach: say so. + +--- + +## 2. Access + +Implements `kumo-data-science/accept/preflight.md` §0. Check, in this order, and stop at the +first that reads the **columns and types** inside `source_scope`. A listing of object names +(`aws s3 ls`, `gcloud storage ls`) proves access, not structure: keep going down the list for a +reader, and if none is found, send the ask-back below with **Details** as the way forward. + +1. **A tool in this session** that reads the system — a server or connector tool for that + warehouse or storage. Use only its read operations. +2. **A command-line tool on the `PATH`** that is already authenticated (table below). Find it + with `command -v `, then run its check command against `source_scope` itself — a + general "who am I" call proves the login, not access to the data. +3. **Python in this session** in which `import kumo.engine` works and the provider's ambient + credentials resolve. Build the connector exactly as the artifact would (`runtime.md` §2) and + read `connector.table_names()` and `connector['
'].columns`, which gives each column's + name and source type. Catalog reads only: no `Table`, `Graph` or snapshot. + +| Source | Tool | Check, inside `source_scope` | Columns and types | Declared keys | §3 probe | +|---|---|---|---|---|---| +| Local files | your file tools; `duckdb`, or Python with `pyarrow` | list the directory | CSV header line; Parquet schema (`duckdb -c "DESCRIBE SELECT * FROM ''"`) | none | `duckdb`, or `pyarrow` | +| S3 | `aws` | `aws s3 ls s3:////` | file names only — not enough on its own; for columns, a reader of `s3://` with the same credentials (`duckdb`, or check 3), else the ask-back | none | the same reader, else skip | +| GCS | `gcloud` | `gcloud storage ls gs:////` | as S3, with `gs://` | none | as S3 | +| BigQuery | `bq` | `bq ls :` | `bq show --schema --format=prettyjson :.
` | `INFORMATION_SCHEMA.TABLE_CONSTRAINTS` and `KEY_COLUMN_USAGE` of the dataset | `bq query --nouse_legacy_sql` | +| Databricks | `databricks` | `databricks tables list ` | `databricks tables get ..
` | the same output's table constraints | only through a SQL warehouse the user named; else skip | +| Snowflake | `snow` | `snow sql -q ""` | the preflight §2 query | `SHOW PRIMARY KEYS` / `SHOW IMPORTED KEYS IN SCHEMA .` | `snow sql -q` | + +A skipped probe leaves its candidates `[inferred, unverified]`, as preflight §0 says. A BigQuery +probe is billed by the bytes it scans; probe only the candidate columns. + +For BigQuery, also read each dataset's `location` with `bq show --format=prettyjson +:` — the source, and the population and destination datasets once confirmed. +A population dataset in a different location from the source cannot be filled by one query: say +so before accepting it. The notebook reads locations itself (`runtime.md` §2), so they are not +config constants. + +**What a check may do.** Read catalog metadata and run the preflight §3 probe — nothing else. +Never run a command that writes, creates, copies or exports data; never sign in, switch a +profile or account, or edit a tool's configuration; never add or update software to get a +reader. Tool permission prompts are the user's to answer; a refused prompt means that path is +unavailable. + +**When nothing reads the source's columns**, send the preflight §0 ask-back. State what you checked and +what happened — "no BigQuery tool in this session; `bq ls my-proj:sales` returned `Access +Denied`" — then offer: + +1. **Access**: authenticate the named command-line tool as an identity that can read the scope, + or connect a tool for that system to this session, and tell you when it is done. The user + does this; you never handle the credential. +2. **Details**: paste the structure. No rows are needed. Useful forms: DDL, `DESCRIBE` output, + `bq show --schema` JSON, a Parquet schema, CSV header lines — with any declared keys. + +--- + +## 3. Preflight + +Answer `kumo-data-science/accept/preflight.md` §1-§6 with the tool §2 found. Write table and +column names exactly as that tool returns them; the artifact must use the same spelling +(`kumoai-reference/references/kumo_finetuning_sdk.md` §3.2). Probe results are evidence for the +proposal only: never written, never saved to a file, never fed to the artifact. + +For files and object stores, a table is a file or a directory of parts in one format, below the +source location. **Name it without its file suffix**: `accounts.parquet` is the table `accounts`. +The connector finds the file from the bare name, and the name you pass becomes the table's name +in the graph, the edges and the PQL, where a suffix's dot does not parse. Show the bare names in +the proposal. + +--- + +## 4. Destinations + +Any destination a Kumo connector writes +(`kumoai-reference/references/kumo_finetuning_sdk.md` §9.2): a table in Snowflake, BigQuery or +Databricks, or a path in a local directory, S3 or GCS. The destination may be in a different +system from the source; it gets its own connector. Not destinations here: ADLS, and BigQuery for +anything but predictions and embeddings. + +A path destination ends in `/` (a directory) or in `.parquet` / `.csv` (one file). The sink adds +`.parquet` to any other path, so the file would not be where the user said. When the user gives +a path with neither, restate it with a trailing `/` at Node 1d and let them confirm or correct it. + +--- + +## 5. Runtime column constraints + +None known for this host. Do not exclude text columns pre-emptively. If a run fails while loading +a text encoder, report the error and the column; that finding belongs at Node 1c. + +--- + +## 6. Capability skills + +The capability skills were written for, and measured on, Snowflake. Their Kumo calls hold for +every connector. Where a step names a Snowflake registry, stage, session or table, follow it only +when the source or destination is Snowflake. For any other connector, say that the step is not +covered on this host; do not adapt it. + +--- + +## 7. Other deployment modes — out of scope + +Hosted Kumo — `kumo.init(...)`, an API key, `KUMO_API_KEY` / `KUMO_API_ENDPOINT`, Kumo Studio — and +the Snowflake Native App are not this SDK. If the user is on one of them, say so and stop. Never +blend their setup into generated code. diff --git a/skills/kumo-data-science/hosts/generic/runtime.md b/skills/kumo-data-science/hosts/generic/runtime.md new file mode 100644 index 000000000..b31cb4f17 --- /dev/null +++ b/skills/kumo-data-science/hosts/generic/runtime.md @@ -0,0 +1,325 @@ +# Host Runtime — Generic coding agent + +Read at Step 2, when generating the artifact: where it runs, the config cell, the connect step, +credentials, destinations and read-back. Intake is in `adapter.md`. + +--- + +## 0. Where the code runs + +Unknown to you, and it does not matter: the artifact must run unchanged in any Python environment +where `import kumo.engine` works and the source's credentials resolve — a workstation, a VM, a +hosted notebook. So it assumes no platform: + +- no Snowpark session from `get_active_session()`, no `dbutils`, no SPCS token, no notebook + magics, no platform secret store; +- no path outside the ones confirmed at intake, except Kumo's scratch default + (`kumoai-reference/references/kumo_finetuning_sdk.md` §2.1). + +Your own session's access (`adapter.md` §2) says nothing about the runtime's. The connect step +resolves credentials where the code runs. + +Write the artifact to the path the user names, else to the current working directory, named after +the task (`churn_30d_train.ipynb`). Write only that file. Creating it is not running it: Step 3 +still waits for an explicit request. + +--- + +## 1. Config cell + +The first code cell holds the imports and every location as a named constant: the confirmed +source, the confirmed destination, and any population table — each written verbatim. + +Values only the runtime knows — a Snowflake warehouse or `connections.toml` entry, a Databricks +SQL warehouse or cluster — are constants here too. S3, GCS and BigQuery have none: the ambient +chain resolves credentials and region where the code runs (an instance or job role, the +environment, `aws configure`, `gcloud`), and BigQuery bills queries to the confirmed dataset's own +project. So add no region, profile, project or billing constant for them beyond the confirmed +location, and pass `region=` or `billing_project_id=` only when the user named one. A BigQuery +dataset's location is not a constant either: the notebook reads it from the dataset (§2). For the +values that are constants, write the user's value if they gave one; otherwise a placeholder of +the form `''`, and list each placeholder in your reply as something to +fill in before running. A placeholder never stands in for a confirmed +value, and never for a secret. + +End the cell with a guard, so a run with an unfilled value stops before reading anything: + +```python +CONFIG = dict(SOURCE_DIR=SOURCE_DIR, OUTPUT_DIR=OUTPUT_DIR) # every constant in this cell +unfilled = [k for k, v in CONFIG.items() if isinstance(v, str) and v.startswith('<')] +assert not unfilled, f'Fill in before running: {unfilled}' +``` + +--- + +## 2. Connect step + +One connector per system, built from the confirmed `source_scope` with ambient credentials +(`kumoai-reference/references/kumo_finetuning_sdk.md` §3, §3.4). Signatures are there; do not +add arguments the user did not confirm. + +```python +# Local files +connector = kumo.FileConnector(root_dir=SOURCE_DIR) +# S3 / GCS — rooted at the confirmed prefix, so table_names() works +connector = kumo.S3Connector(root_dir=SOURCE_URI) # 's3://bucket/prefix/'; no region= +connector = kumo.GCSConnector(root_dir=SOURCE_URI) # 'gs://bucket/prefix/' +# BigQuery — PROJECT_ID and DATASET_ID are the two parts of the confirmed `project.dataset` scope, +# never placeholders. The identity is required; with no arguments it is the ambient Google +# credential. No billing_project_id (§1). location= is the dataset's own, read by bq_location below. +connector = kumo.BigQueryConnector(project_id=PROJECT_ID, dataset_id=DATASET_ID, + identity=kumo.BigQueryIdentity(), + location=bq_location(DATASET_ID)) +# Databricks — exactly one of warehouse_id / cluster_id +connector = kumo.DatabricksConnector( + catalog=CATALOG, schema_name=SCHEMA, identity=kumo.DatabricksIdentity(), + warehouse_id=DATABRICKS_WAREHOUSE_ID) +# Snowflake — outside Snowflake's own runtime, through a connections.toml entry +connector = kumo.SnowflakeConnector.from_connection_toml( + connection_name=SNOWFLAKE_CONNECTION, warehouse=SNOWFLAKE_WAREHOUSE, + database=DATABASE, schema_name=SCHEMA) # lower case: keep the quotes, §3 + +assert connector.table_names(), 'Connector reached no tables.' +``` + +Emit only the line for the confirmed system. Never emit a rootless S3 or GCS connector for a +source: it cannot list tables. + +**BigQuery location.** `BigQueryConnector` defaults to `location='US'`, and a dataset anywhere +else — `EU`, or a single region such as `us-central1`, which is not `US` — fails: reads with +not-found, and the prediction write only after training, at the sink's location check. Whenever +any connector in the notebook is BigQuery — source, population or destination — define this +once, in the connect step before the first connector, and pass +`location=bq_location()` to every `BigQueryConnector`: + +```python +from google.cloud import bigquery + +def bq_location(dataset): + """The location a dataset lives in; the dataset must already exist.""" + return bigquery.Client(project=PROJECT_ID).get_dataset(f'{PROJECT_ID}.{dataset}').location +``` + +Name file and object-store tables without their suffix — `connector['accounts']` for +`accounts.parquet` — and use those names in the tables, edges and PQL (`adapter.md` §3). + +**Credentials.** The ambient forms above name no secret. Use an explicit identity (§3.4) only +when the user asks for one, and then read each secret at run time from an environment variable +whose **name** is a config-cell constant — never a literal, never a value asked for in chat: + +```python +identity = kumo.S3Identity(access_key_id=os.environ[AWS_KEY_ID_VAR], + secret_access_key=os.environ[AWS_SECRET_VAR]) +``` + +The Snowflake factory `from_spcs_env` and Snowflake's notebook secret store belong to +Snowflake's own runtime. Emit them only when the user says the code runs there. + +### 2.1 Storage + +Kumo's scratch directory defaults to `~/.cache/kumo-engine`. Emit `kumo.configure(scratch_dir=...)` +only for a scratch path the user named. A `FileConnector` source or destination must not overlap +it (§3.4). Scratch is disposable: anything that must outlive the run is written to the confirmed +destination. + +--- + +## 3. Destinations and read-back + +Build the output connector for the destination's own system, and the `PredictionOutputConfig` in +that connector's form (`kumoai-reference/references/kumo_finetuning_sdk.md` §9.2). Then read the +destination back and compare its row count to `prediction_result.num_rows` — the only proof the +write worked. + +**A path destination is written at exactly the confirmed path**, never inside it. Root the output +connector at the destination's **parent** and name the destination's last segment in +`output_paths`, keeping its trailing `/` for a directory: + +```python +# confirmed destination: s3://bucket/prediction_outputs/churn_scores/ +OUTPUT_PARENT, OUTPUT_LEAF = 's3://bucket/prediction_outputs/', 'churn_scores/' +output_connector = kumo.S3Connector(root_dir=OUTPUT_PARENT) # FileConnector / GCSConnector alike +output_config = kumo.PredictionOutputConfig( + connector=output_connector, output_paths={'predictions': OUTPUT_LEAF}) +OUTPUT_LOCATION = OUTPUT_PARENT + OUTPUT_LEAF # the confirmed path, for read-back +``` + +A BigQuery destination's connector is scoped to `OUTPUT_DATASET` and takes +`location=bq_location(OUTPUT_DATASET)` (§2), which may differ from the source's. + +Rooting the connector at the destination itself and passing a `table_name` writes one level +below it, to a path the user never confirmed. A directory destination must be empty or absent; +the sink creates it. + +`OUTPUT_LEAF` ends in `/`, `.parquet` or `.csv` (`adapter.md` §4). File, S3 and GCS sinks alike +add `.parquet` to any other leaf, and the read-back would then miss the file. The suffix also +sets the format the sink writes, so read back in that format. + +```python +# Local files, S3, GCS — directory or single-file output +import pyarrow.dataset as ds +OUTPUT_FORMAT = 'csv' if OUTPUT_LOCATION.rstrip('/').lower().endswith('.csv') else 'parquet' +written = ds.dataset(OUTPUT_LOCATION, format=OUTPUT_FORMAT).count_rows() # full path or URI + +# BigQuery +from google.cloud import bigquery +written = next(iter(bigquery.Client(project=PROJECT_ID).query( + f'SELECT COUNT(*) FROM `{PROJECT_ID}.{OUTPUT_DATASET}.{OUTPUT_TABLE}`').result()))[0] + +# Databricks — with the confirmed SQL warehouse; wait_timeout is at most '50s', so poll +import time +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.sql import StatementState + +def q(*parts): + """`catalog`.`schema`.`table`: Unity Catalog names may contain hyphens.""" + return '.'.join('`' + p.replace('`', '``') + '`' for p in parts) + +ws = WorkspaceClient() +r = ws.statement_execution.execute_statement( + warehouse_id=DATABRICKS_WAREHOUSE_ID, wait_timeout='50s', + statement=f'SELECT COUNT(*) FROM {q(CATALOG, OUTPUT_SCHEMA, OUTPUT_TABLE)}') +while r.status.state in (StatementState.PENDING, StatementState.RUNNING): + time.sleep(5) + r = ws.statement_execution.get_statement(r.statement_id) +assert r.status.state == StatementState.SUCCEEDED, r.status +written = int(r.result.data_array[0][0]) + +# Snowflake — through the same connections.toml entry +import snowflake.connector +with snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION) as conn: + written = conn.cursor().execute( + f'SELECT COUNT(*) FROM {OUTPUT_DATABASE}.{OUTPUT_SCHEMA}.{OUTPUT_TABLE}').fetchone()[0] + +assert written == prediction_result.num_rows, (written, prediction_result.num_rows) +``` + +- `OUTPUT_LOCATION` for files, S3 and GCS is the confirmed destination itself. +- A Databricks destination is passed to Kumo as `'__kumo__
'` and read back as + `..
`. With a cluster and no SQL warehouse, there is no read-back here: + say so in the artifact and in your reply. +- Report the destination in its own form, unchanged — `DB.SCHEMA.TABLE`, or the path. + +> **Not verified:** none of these read-backs, and no connector other than Snowflake, has been run +> end to end with this skill. Say so when you hand over an artifact for one of them. + +### 3.1 Static scoring population + +When Node 1d confirmed a population table (`kumoai-reference/references/kumo_finetuning_sdk.md` +§7.1), stage it in the **source's** system — where the entity table is read from — whatever the +destination. Each recipe below reads the entity table and writes the population inside that one +system; the destination may be in another system (`adapter.md` §4) and plays no part here. + +- **Files, S3, GCS source:** read the entity key — and, when the entity table has a time column, that + column too — from the source with `pyarrow.dataset`; name them `ENTITY1` and `TIMESTAMP`; write + one Parquet file inside the confirmed population path, creating the directory first for local + files; and build the `PredictionTable` from a connector rooted there. Decide from the entity's + `Table` as you built it: if it has a `time_column`, the population **must** carry it as + `TIMESTAMP`, **cast to a timestamp**. Kumo rejects a population without it, even for a static + query, whatever the §7.1 Snowflake example shows (`ValueError: ... missing TIMESTAMP`), and + reads a `DATE` anchor as null (`Column 'TIMESTAMP' is date32[day]`): a date column is cast. + + ```python + import os + import pyarrow as pa + import pyarrow.compute as pc + import pyarrow.dataset as ds + import pyarrow.parquet as papq + + keep = {ENTITY_KEY: 'ENTITY1'} + if ENTITY_TIME: # the entity Table's time_column, else None + keep[ENTITY_TIME] = 'TIMESTAMP' + population = ds.dataset(ENTITY_SOURCE, format=ENTITY_FORMAT).to_table(columns=list(keep)) + population = population.rename_columns([keep[c] for c in population.column_names]) + if 'TIMESTAMP' in population.column_names: # a DATE anchor reads as null + i = population.column_names.index('TIMESTAMP') + population = population.set_column(i, 'TIMESTAMP', + pc.cast(population['TIMESTAMP'], pa.timestamp('us'))) + if '://' not in POPULATION_PATH: + os.makedirs(POPULATION_PATH, exist_ok=True) # object stores need no directory + papq.write_table(population, POPULATION_PATH + 'population.parquet') + if POPULATION_PATH.startswith('s3://'): # the source's kind, in the §2 forms + population_connector = kumo.S3Connector(root_dir=POPULATION_PATH) + elif POPULATION_PATH.startswith('gs://'): + population_connector = kumo.GCSConnector(root_dir=POPULATION_PATH) + else: + population_connector = kumo.FileConnector(root_dir=POPULATION_PATH) + prediction_table = kumo.PredictionTable.from_source_table( + population_connector['population'], pq=pquery, snapshot=snapshot) + ``` + + `ENTITY_SOURCE` is the entity table's file or directory as preflight listed it, and + `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. The population is + written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the + source's file system or bucket. +- **Snowflake source:** the §7.1 SQL, run through + `snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION)` as in the read-back above, + and the `PredictionTable` built from a `SnowflakeConnector` made as in §2 but scoped to the + population's database and schema. +- **Databricks source:** create the table through the source's SQL warehouse with the statement + API, wait for it, and build the `PredictionTable` from a `DatabricksConnector` scoped to the + population's schema. A source reached through a cluster has no SQL warehouse: say at Node 1d + that staging is not covered there, and offer the rows whose target is NULL instead. The same `TIMESTAMP` rule applies: project `CAST(
']` tries the +exact name, then its upper-cased form, so `connector['ENTITY_PROFILE']` never finds a lower-case +`entity_profile` (`SourceNotFoundError: Invalid table ENTITY_PROFILE: it has no columns`). Write +table names as `table_names()` returns them and column names as `.columns` returns them — in +`primary_key=`, `time_column=`, column lists and filters. + +**Never exclude a column by a case-sensitive `!=` against a hand-typed literal.** A filter such as +`c.name != 'END_DATE'` silently keeps a lower-case `end_date` — the comparison never matches, the +column is never dropped, and nothing raises. Pass the confirmed `Excluded columns` from preflight +(`accept/preflight.md` §8) as a literal `column_names=` list built from the catalog's own spelling +— it has no hand-typed comparison to get wrong. If a filter is used instead, compare +case-insensitively: `c.name.lower() != 'end_date'.lower()`. + +Source: shipped package (`kumo.engine.connector.snowflake`, `.base`, `.source_table`; +`kumo_snowflake.identity` for the lookup order) + +### 3.3 The Snowpark session: take it once, first + +Kumo's Snowflake connector opens its own Snowpark session. Once Kumo has read or written +Snowflake, the process holds two sessions, and a later `get_active_session()` raises +`SnowparkSessionException: (1409): More than one active session is detected`. + +```python +from snowflake.snowpark.context import get_active_session + +session = get_active_session() # first code cell, before any connector +``` + +- Call `get_active_session()` **once**, in the first code cell, before building any connector. + Reuse that `session` for every later Snowflake call: the write-back check (§9.1) and any + population table (§7.1). +- Never call `get_active_session()` again later in the notebook, and never open another session + with `Session.builder`. + +Source: shipped package (`kumo_snowflake.session`, which opens its session with +`Session.builder.configs(...).create()`) + +### 3.4 Every other connector + +Constructors and identities, all keyword-only. Source: shipped package +(`kumo.engine.connector.file`, `kumo.engine.connector.s3`, `kumo.engine.connector.gcs`, +`kumo.engine.connector.bigquery`, `kumo.engine.connector.databricks`, +`kumo.engine.connector.identity`): + +``` +FileConnector(*, root_dir) +S3Connector(*, root_dir=None, identity=None, region=None) +GCSConnector(*, root_dir=None, identity=None, requester_pays_project=None) +BigQueryConnector(*, project_id, dataset_id, identity=None, location='US', + billing_project_id=None, staging_uri=None, require_staged=False) +DatabricksConnector(*, catalog, schema_name, identity, warehouse_id=None, cluster_id=None, + staging_volume_path=None, require_staged=False) +DatabricksConnector.from_notebook(*, catalog, schema_name, warehouse_id=None, cluster_id=None, + staging_volume_path=None, require_staged=False) + +S3Identity(*, access_key_id=None, secret_access_key=None, session_token=None, role_arn=None, + partition='aws', connection=None) +GCSIdentity(*, service_account_key=None, workload_service_account_email=None) +BigQueryIdentity(*, service_account_key=None, workload_service_account_email=None) +DatabricksIdentity(*, host=None, client_id=None, client_secret=None) +``` + +**Prefer ambient credentials.** Each connector resolves its provider's own credential chain when +no secret is passed, so the connect step names no secret at all: + +| Connector | Ambient form | Credentials it resolves | Table names | `sql()` | +|---|---|---|---|---| +| `FileConnector` | `FileConnector(root_dir=...)` | none needed | relative paths below `root_dir`, optionally with `.csv` / `.parquet`, or a directory of parts in one format | no | +| `S3Connector` | `identity=None` | AWS chain: instance profile, `aws configure`, environment | relative paths below `root_dir`; with `root_dir=None`, full `s3://` URIs and no `table_names()` | no | +| `GCSConnector` | `identity=None` | Application Default Credentials | as S3, with `gs://` | no | +| `BigQueryConnector` | `identity=None` | Application Default Credentials | tables of `dataset_id` | yes | +| `DatabricksConnector` | `from_notebook(...)`, or `identity=DatabricksIdentity()` | the Databricks SDK's chain: a notebook's context, a CLI profile, environment | tables of `catalog.schema_name` | yes | + +- The explicit identity forms carry secrets (an access key, a service-account key dict, an OAuth + client secret). Use one only when the user asks for it, and read each secret from the + environment's secret store at run time — never a literal, never a value asked for in chat. +- `FileConnector.root_dir` is a filesystem directory, not a URI, and must not overlap the + scratch directory (§2.1). Absolute table paths, URIs, `..` and globs are rejected. A CSV time + column needs `.with_column(col, dtype=Dtype.date, timestamp_format=...)`. +- `region` on `S3Connector` is optional and normally **omitted**: the ambient chain resolves the + bucket's region where the code runs. Pass it only when the user named one, and only with + `root_dir`. It is never a value to ask for, and never a placeholder. +- `DatabricksConnector` takes exactly one of `warehouse_id` and `cluster_id`. +- `require_staged=True` needs `staging_uri` (BigQuery) or `staging_volume_path` (Databricks). +- ADLS has no connector in this package. + +Where each one writes is §9.2. + +--- + +## 4. Tables + +`Table` is **immutable**. `with_column(...)` returns a *new* table — assign or chain it, never +call it for its side effect. + +```python +from kumoapi.typing import Dtype, Stype + +account_profile = kumo.Table.from_source_table( + connector['account_profile'], primary_key='account_id', time_column='created_date', +) + +transaction_events = ( + kumo.Table.from_source_table( + connector['transaction_events'], primary_key='event_id', time_column='event_date', + ) + .with_column('account_id', stype=Stype.ID) + .with_column('device_id', stype=Stype.ID) +) +``` + +Source: shipped package (`kumo.engine.table`): + +``` +Table.from_source_table(source_table, column_names=None, *, primary_key=None, + time_column=None, end_time_column=None) +Table.with_column(name, *, stype=None, dtype=None, timestamp_format=None, + multicategorical_delimiter=None) +Table.infer_metadata() +``` + +- **The source table and column list are positional**; everything else is keyword-only. +- Omit `column_names` to take every column. Pass a list to restrict. +- **Foreign keys infer as numerical.** FK-ness is a graph-level fact, so per-table inference sees + only values. Correct every FK with `.with_column(col, stype=Stype.ID)`. +- **A TEXT time column needs `dtype=Dtype.date`** declared, or staging will not cast it. +- `infer_metadata()` takes no arguments. +- Read-back attributes live under `.definition`: `.definition.pkey`, `.definition.time_col`, + `.definition.end_time_col`. `.table_name` is on the table itself. + +> **Not verified:** `timestamp_format` and `multicategorical_delimiter` are in the signature but +> exercised nowhere. + +Source: shipped package (`kumo.engine.table`) + +--- + +## 5. Graph + +```python +from kumo.engine import Edge + +graph = kumo.Graph( + tables=[account_profile, transaction_events, device_profile], + edges=[ + Edge(src_table='transaction_events', fkey='account_id', dst_table='account_profile'), + Edge(src_table='transaction_events', fkey='device_id', dst_table='device_profile'), + ], +) +graph.validate() +``` + +- **Emit `tables` as a list.** Each table's graph name is then its source table name + (`.table_name`), which is what edges and PQL reference. +- `tables` may also be a mapping `{alias: table}`. The alias then replaces the source name in edges + and PQL; the connector still reads the source name. It is valid in a user's notebook — do not + call it superseded — but emit the list form unless two tables share a source name, which the + list form rejects with `Duplicate graph table name`. +- **`edges` are `Edge` objects** from `kumo.engine`, not dicts or tuples. +- Edges point **foreign key -> primary key**: `src_table.fkey` references `dst_table`'s primary key. +- **`graph.validate()` takes no arguments.** There is no `verbose=`. +- Emit `validate()` explicitly. It is a structural check needing only schemas — no snapshot yet, + so it is cheap and catches edge typos before any data moves. + +Surface includes `.tables`, `.edges`, `.id`, `.native_graph`, `.validate()`, `.visualize()`, `.snapshot()`. +**There is no `add_table`, `remove_table`, `link`, `unlink`, `infer_links`, `get_edge_stats`, or +`get_table_stats`.** + +`Graph.visualize(path=None, *, show_cols=False, show_samples=False, layout='auto')` displays or exports the graph. +Follow `kumo-graph-visualization/SKILL.md` for generated notebook visuals, script exports, and schema inspection during model iteration. + +Source: shipped package (`kumo.engine.graph`) + +--- + +## 6. Predictive query + +```python +pquery = kumo.PredictiveQuery(graph=graph, query=QUERY) +pquery.validate() +print('task type:', pquery.get_task_type()) +``` + +- The graph is bound at construction; parsing is lazy. +- **Emit `validate()` with no arguments.** It raises on an invalid query; otherwise it returns a + `ValidationResponse` and logs its warnings. +- **Always emit `validate()` then `get_task_type()`** before training. The task type is never + written in the query — it is inferred from the target's shape — so print it and confirm it + matches intent. A query that validates can still be the wrong task. +- Print `.query` back so the user can check the generated PQL. + +See `kumoai-reference/references/pql_writing.md` for the query language itself. + +Source: shipped package (`kumo.engine.pquery.predictive_query`) + +--- + +## 7. Snapshot, training and prediction tables + +Materialization is **explicit**. `fit` does not do it for you. + +```python +snapshot = graph.snapshot() + +training_table = pquery.generate_training_table(snapshot) +prediction_table = pquery.generate_prediction_table( + snapshot, prediction_time=PREDICTION_TIME, +) +``` + +Source: shipped package (`kumo.engine.graph`, `kumo.engine.pquery.predictive_query`): + +``` +Graph.snapshot() +PredictiveQuery.generate_training_table(snapshot, plan=None) +PredictiveQuery.generate_prediction_table(snapshot, *, prediction_time=None, plan=None) +``` + +- `snapshot` is **positional** on both. `prediction_time` and `plan` are keyword-only on + `generate_prediction_table`. +- `graph.snapshot()` reads every table through its connector and builds an on-disk DiskGraph index + under the scratch directory. An unchanged valid snapshot is reused on later runs. +- `GraphSnapshot` exposes `.id`, `.index_path`, `.graph_config`. +- `TrainingTable` exposes `.train_table_path` (ordinary parquet), `.manifest_path`, + `.required_stages`, `.generation_plan`. +- `PredictionTable` exposes `.table_path`, `.row_count`, `.timestamps`, `.required_stages`. +- **`prediction_time` must be a NAIVE `datetime` — passing a timezone-aware one raises** + `ValueError: prediction_time must not include a timezone`. Kumo applies UTC itself + (`kumo_core.prediction_options.prediction_timestamps`), so write `datetime(2026, 8, 1)` and + **never** `datetime(2026, 8, 1, tzinfo=timezone.utc)`. Do not import `timezone` for this. +- It must not be earlier than the start of the data's event history (an earlier anchor raises + `Invalid Prediction Anchor Time`; a later date is accepted), and it **cannot be computed from + the connector** — `connector.sql(query)` returns a source table to feed `Table.from_source_table`, + not rows. Hardcode a known in-range value and say why. Omitted, a temporal query anchors at + the latest source time of its target; state that choice if you rely on it. +- `prediction_time` is rejected outright for a **static** query — it is temporal-only. + +Source: shipped package (`kumo.engine.pquery.predictive_query`, +`kumo.engine.pquery.prediction_table`, `kumo_core.prediction_options`, `kumo_core.training.validation`) + +### 7.1 A static query scores the rows whose target is NULL + +In prediction mode a static query keeps only the rows whose target is NULL: Kumo trains on the +labeled rows and scores the unlabeled ones. If every row already has its target, +`generate_prediction_table` raises `RuntimeError: prediction query produced no rows`. + +The scoring population confirmed at the task gate (`kumo-data-science/SKILL.md` Node 1d) decides +the path. The staging example below is Snowflake's; on any other connector the host's `runtime.md` +says how a population is staged, or that it is not covered there: + +- **Rows whose target is NULL** → `pquery.generate_prediction_table(snapshot)`, no + `prediction_time`. +- **Rows that already have a target** (for example "every order") → stage their keys as + `ENTITY1` in the confirmed population table and pass it explicitly: + +```python +session.sql( # the session from §3.3 + f'CREATE OR REPLACE TABLE {POP_DATABASE}.{POP_SCHEMA}.{POP_TABLE} AS ' + f'SELECT "{ENTITY_KEY}" AS ENTITY1 FROM {SOURCE_DATABASE}.{SOURCE_SCHEMA}."{ENTITY_TABLE}"' +).collect() +population_connector = kumo.SnowflakeConnector.from_spcs_env( # or from_connection_toml, §3 + warehouse=WAREHOUSE, database=POP_DATABASE, schema_name=POP_SCHEMA, +) +prediction_table = kumo.PredictionTable.from_source_table( + population_connector[POP_TABLE], pq=pquery, snapshot=snapshot, +) +``` + +- Stage the population only where Node 1d confirmed it — never in the source schema. Write + `POP_TABLE` in upper case, so the unquoted `CREATE` and the connector lookup agree. +- Never change the query, the entity or the target to get rows. An empty population is a fact + about the data to report, not a query to rewrite. +- The population table must carry `ENTITY1` and, when the task samples against seed times (a + temporal query, or a static one on a timestamped entity), a `TIMESTAMP` column; it must not + contain a target column. The SQL above stages only `ENTITY1`, which suffices for a static query + on an entity without a timestamp; otherwise also select the anchor time as `TIMESTAMP`. +- Rows the model trained on are scored too. Say so: those predictions are not out-of-sample. + +Source: shipped package (`kumo_core.prediction.generate_prediction_table`, +`kumo.engine.pquery.prediction_table.PredictionTable.from_source_table`) + +--- + +## 8. Model plan and training + +```python +from kumoapi.model_plan import RunMode + +model_plan = pquery.suggest_model_plan(snapshot, run_mode=RunMode.FAST) + +# Smoke-test sizing — say so, and remove for a real run. +model_plan.training_job.num_experiments = 1 +model_plan.training_job.refit_trainval = False +model_plan.training_job.refit_full = False +model_plan.optimization.max_epochs = 3 + +trainer = kumo.Trainer(model_plan) +training_result = trainer.fit(training_table, snapshot=snapshot) +print(training_result.metrics) +``` + +Source: shipped package (`kumo.engine.pquery.predictive_query`, `kumo.engine.trainer.trainer`): + +``` +PredictiveQuery.suggest_model_plan(snapshot, *, run_mode=RunMode.FAST, train_table_spec=None) +Trainer(model_plan) +Trainer.fit(train_table, *, snapshot, export_config=None, warm_start=None) +``` + +- **`suggest_model_plan` is implemented.** Prefer it over hand-building a plan: it inspects the + materialized graph and query. `snapshot` is positional; `run_mode` is keyword-only. +- **`fit` requires `snapshot=` as a keyword.** `fit(training_table)` alone is a `TypeError`. This + is the single easiest signature to get wrong. +- Mutate the returned plan in place for sizing. Always tell the user when numbers are smoke-test + values rather than a real run. +- `TrainingResult` exposes `.metrics` (a `Dict[str, float]` **attribute**, not a call), + `.artifact_paths`, `.bundle_dir`, `.published_model`. +- `export_config=ModelOutputConfig(...)` publishes the bundle during `fit`. +- Success is a blocking call returning without raising. **There is no job status enum — never + branch on one.** + +`Trainer` also exposes `.model_plan` (settable), `.training_table_generation_plan`, `.scratch_dir`, +and reloading: `Trainer.load(bundle_path, embedding_models=None)` and +`Trainer.load_from_connector(model_registry, *, path=None, location=None, +embedding_models=None)`. + +Source: shipped package (`kumo.engine.trainer.trainer`) + +--- + +## 9. Prediction and write-back + +```python +from kumoapi.jobs import PredictionArtifactType, WriteMode + +output_connector = kumo.SnowflakeConnector.from_spcs_env( + warehouse=WAREHOUSE, database=OUTPUT_DATABASE, schema_name=OUTPUT_SCHEMA, +) +output_config = kumo.PredictionOutputConfig( + connector=output_connector, + artifact_type=PredictionArtifactType.PREDICTIONS, + table_name=output_table, + write_mode=WriteMode.OVERWRITE, +) + +prediction_result = trainer.predict( + prediction_table, snapshot=snapshot, output_config=output_config, +) +print('rows:', prediction_result.num_rows) +``` + +Source: shipped package (`kumo.engine.trainer.trainer`, `kumo.engine.trainer.output_config`): + +``` +Trainer.predict(prediction_table, *, snapshot, output_config=None, + num_classes_to_return=None, num_workers=1, + binary_classification_threshold=None) + +PredictionOutputConfig(connector=, artifact_type=, table_name=, write_mode=, extra_fields=, + output_paths=None) +PredictionResult.export(output_config) +``` + +- For Snowflake, always pass `artifact_type` and `table_name`. `output_paths` is for file and + object-store connectors only; with a `SnowflakeConnector` it raises `ValueError`. Every + connector's form is in §9.2. +- `extra_fields=[MetadataField.JOB_TIMESTAMP, MetadataField.ANCHOR_TIMESTAMP]` (`kumoapi.jobs`) + adds those two columns: when the run started, and the time each prediction is anchored at. + `ANCHOR_TIMESTAMP` exists only for a query with a time window; requesting it for one without + raises `ValueError` after scoring. Source: shipped package (`kumo_core.train.output`, + `kumo_core.prediction_options`). + +- **`snapshot=` is required.** `output_config` is optional — omit it to keep results local. +- `predict` requires a trainer holding fitted state — from `fit` on that same `Trainer`, or from + `Trainer.load` / `Trainer.load_from_connector`; there is no job registry to look a model up in. +- `num_classes_to_return` is the number of results returned per entity, and it is independent + of the query's `RANK TOP K`: a `RANK TOP 10` model can return 50 results. Link prediction + requires it whenever predictions are written, as an integer from 1 to 500 + (`kumo_core.prediction_options.MAX_LINKS_TO_PREDICT`); for more than 500 per entity, export + embeddings instead. Multiclass and multilabel accept it optionally and fall back to the + query's `RANK TOP K` when it has one. Regression and binary classification reject it. +- Binary classification requires an explicit `binary_classification_threshold`; there is no SDK + default. Use the one confirmed at Node 1d (§12). +- `binary_classification_threshold` and `num_classes_to_return` are arguments of `predict`, and + only of `predict`. `PredictionOutputConfig` describes where to write, not how to predict: it + takes neither, and passing one is a `TypeError`. Whatever the destination, the config carries + only the fields in its signature above. +- `PredictionResult` exposes `.num_rows`, `.files`, `.frame`, `.selection`, `.artifact_type`. + Prefer `.num_rows` and `.files` — `.frame` collects every row into pandas. +- `prediction_result.export(output_config)` writes an existing result to a destination. Passing + `output_config=` to `predict` does the same write in one call; that is the form to emit. + +### 9.1 The output table keeps its name + +**The table is written with exactly the name you pass.** No suffix, no case change, no quoting +needed on the write. Verify by reading it back and comparing the row count. Quote the table name +in the read: `session.table` upper-cases an unquoted part, so `churn_scores` would look for +`CHURN_SCORES` and not find the table Kumo wrote. The database and schema stay unquoted; both +sides resolve those the same way. + +```python +def quoted(name): + return '"' + name.replace('"', '""') + '"' + +destination = session.table([OUTPUT_DATABASE, OUTPUT_SCHEMA, quoted(output_table)]) +assert destination.count() == prediction_result.num_rows +``` + +Scope the output connector to the destination database and schema; `table_name` is a bare name. +Always read the table back — it is the only proof the write-back worked. `session` is the one +taken in the first cell (§3.3); calling `get_active_session()` here, after Kumo has written, +raises. + +Source: shipped package (`kumo.engine.trainer`) + +### 9.2 Destinations by connector + +The output connector may differ from the source connector. Source: shipped package +(`kumo.engine.trainer.output_config`, `kumo.engine.connector.file`, `kumo_core.object_store_sink`, +`kumo_databricks.sink`, `kumo_databricks.output`): + +| Connector | `PredictionOutputConfig` form | Rules | +|---|---|---| +| `SnowflakeConnector` | `artifact_type=` + `table_name=` (bare name) | connector scoped to the destination database and schema; §9.1 | +| `FileConnector`, `S3Connector`, `GCSConnector` | `output_paths={'predictions': ..., 'embeddings': ...}` — the keys select the artifacts — or `artifact_type=` + `table_name=` | do not combine the two forms; a path ending in `/` is a directory and must be empty; otherwise one file, written in the format its `.parquet` or `.csv` suffix names, and `.parquet` is added when it has neither — by all three connectors; paths are relative to a rooted connector, full URIs for a rootless one | +| `BigQueryConnector` | `artifact_type=` + `table_name=` (unqualified table ID in its dataset) | predictions and embeddings only — not a model `export_config` and not a `Trainer.load_from_connector` source | +| `DatabricksConnector` | `artifact_type=` + `table_name=`: a bare name, or `'__kumo__
'` | a bare name lands in the connector's schema; a qualified name must name the connector's own schema, or it is refused; writes stage through the volume `.._kumo_staging`, created on first use | + +`output_paths` with a non-file connector raises `ValueError`. Read every destination back after +the write, as the host's `runtime.md` says (§9.1 for Snowflake). + +--- + +## 10. Complete verified example + +On Snowflake. On any other connector, replace steps 1, 7 and 8 with the host `runtime.md`'s +connect step, the §9.2 destination and its read-back; nothing else changes. + +Self-contained: `OUTPUT_DATABASE`, `OUTPUT_SCHEMA` and `OUTPUT_TABLE` are placeholders for the +confirmed output destination (§12). + +```python +from datetime import datetime +from pathlib import Path + +import kumo.engine as kumo +from kumo.engine import Edge +from kumoapi.jobs import PredictionArtifactType, WriteMode +from kumoapi.model_plan import RunMode +from kumoapi.typing import Stype +from snowflake.snowpark.context import get_active_session + +OUTPUT_DATABASE, OUTPUT_SCHEMA, OUTPUT_TABLE = '', '', '
' # confirmed destination + +session = get_active_session() + +# 1. Scratch directory, then connect (inside SPCS; elsewhere use the §3 order). No credentials in the notebook. +kumo.configure(scratch_dir=Path('/tmp/kumo-run') / 'scratch') +connector = kumo.SnowflakeConnector.from_spcs_env( + warehouse='WH_S', database='SYNTHETIC_DATASETS', schema_name='"my_schema"', +) +assert connector.table_names(), 'Connector reached no tables.' + +# 2. Tables — correct every foreign key to Stype.ID. +account_profile = kumo.Table.from_source_table( + connector['account_profile'], primary_key='account_id', time_column='created_date', +) +transaction_events = ( + kumo.Table.from_source_table( + connector['transaction_events'], primary_key='event_id', time_column='event_date', + ) + .with_column('account_id', stype=Stype.ID) + .with_column('location_id', stype=Stype.ID) +) +location_profile = kumo.Table.from_source_table( + connector['location_profile'], primary_key='location_id', +) + +# 3. Graph — list of tables, Edge objects, explicit validate. +graph = kumo.Graph( + tables=[account_profile, transaction_events, location_profile], + edges=[ + Edge(src_table='transaction_events', fkey='account_id', dst_table='account_profile'), + Edge(src_table='transaction_events', fkey='location_id', dst_table='location_profile'), + ], +) +graph.validate() + +# 4. Query — validate, then confirm the inferred task type. +pquery = kumo.PredictiveQuery( + graph=graph, + query=('PREDICT LIST_DISTINCT(transaction_events.location_id, 0, 30, days) ' + 'RANK TOP 10 FOR EACH account_profile.account_id'), +) +pquery.validate() +print('task type:', pquery.get_task_type()) + +# 5. Materialize. PREDICTION_TIME is NAIVE — a tz-aware datetime raises ValueError. +PREDICTION_TIME = datetime(2026, 8, 1) +snapshot = graph.snapshot() +training_table = pquery.generate_training_table(snapshot) +prediction_table = pquery.generate_prediction_table( + snapshot, prediction_time=PREDICTION_TIME, # NAIVE datetime, inside the event history +) + +# 6. Train — snapshot= is required. +model_plan = pquery.suggest_model_plan(snapshot, run_mode=RunMode.FAST) +model_plan.optimization.max_epochs = 3 # smoke-test sizing +trainer = kumo.Trainer(model_plan) +training_result = trainer.fit(training_table, snapshot=snapshot) +print(training_result.metrics) + +# 7. Predict and write back — snapshot= is required here too. +output_connector = kumo.SnowflakeConnector.from_spcs_env( + warehouse='WH_S', database=OUTPUT_DATABASE, schema_name=OUTPUT_SCHEMA, +) +prediction_result = trainer.predict( + prediction_table, + snapshot=snapshot, + output_config=kumo.PredictionOutputConfig( + connector=output_connector, + artifact_type=PredictionArtifactType.PREDICTIONS, + table_name=OUTPUT_TABLE, + write_mode=WriteMode.OVERWRITE, + ), +) +print('rows:', prediction_result.num_rows) + +# 8. Verify the write-back. The table name is quoted (§9.1). +destination = session.table([OUTPUT_DATABASE, OUTPUT_SCHEMA, '"' + OUTPUT_TABLE.replace('"', '""') + '"']) +assert destination.count() == prediction_result.num_rows +``` + +Source: shipped package (`kumo.engine`, `kumoapi`, `kumo_snowflake.spcs`) + +--- + +## 11. Never emit + +### 11.1 The old SDK — recognise and refuse + +These were the **previous** Kumo SDK. They are gone from the current package; each one is an +`AttributeError` or `TypeError`. If a user's existing notebook contains them, say plainly that it +targets a superseded build and needs rewriting against §10 — do not copy its shape. + +| Old | Now | +|---|---| +| `kumo.set_backend(...)`, `kumo.LocalBackend(...)`, `kumo.get_backend()` | nothing — there is no backend | +| `kumo.Storage(...)`, `kumo.FilesystemArtifactLocation(...)` | `kumo.configure(scratch_dir=...)` (§2.1) | +| `kumo.create_identity(CreateIdentityArgs(...))`, `kumo.test_identity_credentials(...)` | `SnowflakeConnector.from_spcs_env()`, else `from_connection_toml()`, else `SnowflakeIdentity` from secrets (§3) | +| `SnowflakeConnector(name=..., account=..., credentials={...})` | keyword-only ctor, no `name`/`account` (§3.1) | +| `kumo.Table(source_table=..., primary_key=...)` | `Table.from_source_table(...)` (§4) | +| `table['COL'].stype = kumo.Stype.ID` | `.with_column('COL', stype=Stype.ID)` (§4) | +| `kumo.Stype`, `kumo.Dtype` | `from kumoapi.typing import Dtype, Stype` | +| `kumo.Graph(..., edges=[{...}])` — edges as dicts | `Edge` objects (§5). A `tables={alias: t}` mapping is still valid | +| `trainer.fit(graph, pquery=...)` | `fit(train_table, snapshot=snapshot)` (§8) | +| `OutputConfig` from `kumoai.artifact_export` | `kumo.PredictionOutputConfig` (§9) | +| `.metrics()` as a call, `.job_id`, `.summary()` | `.metrics` attribute, `.num_rows` (§8, §9) | +| output table renamed to `"_predictions"` | the name is kept verbatim (§9.1) | + +### 11.2 Hosted Kumo — out of scope entirely + +- `kumo.init(...)` in any form (`url=`, `api_key=`, `snowflake_credentials=`, + `snowflake_application=`), API keys, `KUMO_API_KEY` / `KUMO_API_ENDPOINT`, Kumo Studio. +- `training_job.tracking_url` or any Kumo UI reference. +- `kumo.Session(...)`, `kumo.ComputeType`, RBAC groups, `project_id`. +- The Snowflake Native App and Marketplace deployment. +- `kumoai.codegen`, `DistillationTrainer`, online serving. + +### 11.3 Also never + +- **Any environment-setup step.** Kumo is already present (§2); begin at `import kumo.engine`. +- Branching on a Kumo job status string or enum — Kumo has none. A Snowflake ML Job's status + is Snowflake's, read from `snowflake.ml.jobs`, and only the host's job file uses it. +- `connector['X'].head(...)` — it does not exist. `connector.sql(query)` exists but returns a + custom-SQL source table, never rows; do not use it to read data (Source: shipped package + (`kumo.engine.connector.source_table`)). +- **`import kumoai`.** That is the hosted SDK's name, and this SDK's name before the rename. Where + it resolves at all, it is the wrong package or a pre-rename build; import `kumo.engine`. + +--- + +## 12. Parameters to inject from the validated request + +| Code location | Source | +|---|---| +| The connect step | The host's `runtime.md`, from the confirmed `source_scope` (§3, §3.4) | +| `TARGET = dict(warehouse=, database=, schema_name=)` (Snowflake) | Data location from the intake contract; a lower-case name keeps its double quotes (§3) | +| `from_connection_toml(connection_name=)` (Snowflake) | The user's `[connections.]` entry; ask | +| `SnowflakeIdentity(account=, username=)`, secret names (Snowflake) | Account identifier, key-pair user, and the names of the attached key and passphrase secrets; ask only when neither factory applies, never for a secret's value | +| `configure(scratch_dir=)` | A local path; `/tmp/` is fine | +| `SnowflakeConnector(role=)` (Snowflake) | Role from the intake contract | +| `connector['TABLE']` names, column names | Exactly as `table_names()` / `.columns` return them (§3.2) | +| Column selection, `stype` / `dtype` | Data structure from the intake contract | +| `Edge(src_table=, fkey=, dst_table=)` | Relationships from the intake contract | +| PQL `query=` | Generated per `kumoai-reference/references/pql_writing.md` | +| `prediction_time=` | A known in-range **naive** `datetime`; **cannot be computed** | +| `predict(num_classes_to_return=N)` | Results per entity the user asked for; defaults to `RANK TOP K` but may differ. Link prediction writing predictions requires it, 1–500; no cap otherwise (§9) | +| `predict(binary_classification_threshold=)` | Required for binary classification; never on `PredictionOutputConfig`. The threshold restated at Node 1d: the user's, else `0.5 [default]` (`kumo-data-science/accept/intake_contract.md`) | +| Population table (static task, rows that already have a target) | Confirmed at Node 1d (§7.1) | +| `PredictionOutputConfig(...)` + output connector | Output destination in its connector's form (§9.2); report it back unchanged — `DB.SCHEMA.TABLE`, or the path | + +--- + +## 13. Pre-return checklist + +- [ ] Generated code begins at `import kumo.engine as kumo`; `import kumoai` appears nowhere + (§11.3). +- [ ] `kumo.init(...)` appears nowhere; no API keys; no `tracking_url`. +- [ ] No call from §11.1 appears anywhere. +- [ ] `Stype` / `Dtype` imported from `kumoapi.typing`; `Column` / `Edge` from `kumo.engine`; + `PredictionArtifactType` / `WriteMode` from `kumoapi.jobs`; `RunMode` from + `kumoapi.model_plan`. +- [ ] The connect step is the host `runtime.md`'s, with no credential literal (§3, §3.4). +- [ ] Snowflake: connection tries `from_spcs_env`, then `from_connection_toml`, then + `SnowflakeIdentity` from secrets (§3), with explicit `warehouse`, `database`, + `schema_name`; `SPCS_TOKEN_PATH` imported, never hardcoded. +- [ ] Snowflake: lower-case or mixed-case `database` / `schema_name` passed with their double + quotes (§3). +- [ ] Table and column names in the catalog's exact spelling (§3.2). +- [ ] Snowflake: `session = get_active_session()` once, in the first code cell; never again, no + `Session.builder` (§3.3). +- [ ] `connector.table_names()` asserted before building tables. +- [ ] Every foreign key corrected with `.with_column(col, stype=Stype.ID)`. +- [ ] `with_column(...)` results assigned or chained — it returns a new Table. +- [ ] `Graph(tables=[...], edges=[Edge(...)])` — list form; `graph.validate()` emitted. +- [ ] `pquery.validate()` and `get_task_type()` run before training, both called with no arguments. +- [ ] `graph.snapshot()` before generating any table. +- [ ] **`trainer.fit(training_table, snapshot=snapshot)`** — `snapshot=` present. +- [ ] **`trainer.predict(prediction_table, snapshot=snapshot, ...)`** — `snapshot=` present. +- [ ] `binary_classification_threshold` / `num_classes_to_return` on `predict`, never on + `PredictionOutputConfig` (§9). +- [ ] `prediction_time` hardcoded and **naive** — no `tzinfo`, no `timezone.utc` — with the reason stated. +- [ ] Static query: generated population only for rows whose target is NULL; otherwise the + population table confirmed at Node 1d (§7.1). +- [ ] No code cell writes anywhere unless a destination was confirmed; an example of how to add + one later is markdown text, not a code cell. +- [ ] Smoke-test sizing called out as such. +- [ ] Destination in its connector's form (§9.2), reported with its exact name — no + `_predictions` suffix. +- [ ] Written destination read back and row count compared to `prediction_result.num_rows`. +- [ ] Success judged by blocking calls returning without raising — not by a status value. + +--- + +## 14. Known gaps + +**Not verified:** + +- `Table`'s `timestamp_format` and `multicategorical_delimiter` — in the signature, exercised + nowhere. +- `train_table_spec=` on `suggest_model_plan`; `plan=` on `generate_training_table` / + `generate_prediction_table`. +- `export_config=` on `fit`, and `ModelOutputConfig`'s `path` vs `location`. +- `Trainer.load` / `load_from_connector` round-trips — implemented in the package, exercised + nowhere. `load_from_connector` needs a connector whose sink implements `ModelFetchingSink` + (otherwise `TypeError`); `FileConnector` provides `fetch_model`, other sinks are unverified. +- `num_workers` above 1. +- `PredictionResult.export` against Snowflake — in the package, exercised nowhere. +- Whether a snapshot can be reused across graphs or persisted between processes. +- The written column schema for a ranking task. +- `FileConnector`, `S3Connector`, `GCSConnector`, `BigQueryConnector`, `DatabricksConnector`: + §3.4 and §9.2 are read from the package source; none has been run end to end with this skill. + +**Known absent** — do not reach for these: `SourceTable.head(...)`, +`graph.infer_links()`, `graph.get_edge_stats()`, `graph.get_table_stats()`, `table.validate()`, +`table.metadata`, `table.get_stats()`, `graph.validate(verbose=)`, any job-status enum. diff --git a/skills/kumo-data-science/kumoai-reference/references/outputs.md b/skills/kumo-data-science/kumoai-reference/references/outputs.md new file mode 100644 index 000000000..9e7d5689a --- /dev/null +++ b/skills/kumo-data-science/kumoai-reference/references/outputs.md @@ -0,0 +1,100 @@ +# Predictions, embeddings and destinations + +`Trainer.predict` selects one artifact per call. Without `output_config`, it produces local +prediction Parquet files. `num_rows` is the selected output row count; `.frame` collects all +files into pandas and should be used only for bounded results. Local files require explicit +retention to survive runtime teardown. + +Inspect schema without collecting all rows: + +```python +import pyarrow.parquet as parquet + +for filename in predictions.files: + print(filename, parquet.ParquetFile(filename).schema_arrow) +``` + +Binary/regression results have scalar output columns. Multiclass/multilabel and rankings +use long class rows. Forecasts use expanded timestamp rows. Embeddings include vector-valued +EMBEDDING columns and entity metadata; link embeddings can include ID, TYPE and NAME for +self-links. Preserve the actual schema and task rather than assuming one row per entity. + +## Write predictions + +Inputs: `output_connector` scoped to the confirmed destination database/schema, exact bare +`OUTPUT_TABLE`, explicit `WRITE_MODE`, and task-specific `PREDICT_OPTIONS`. WriteMode has +APPEND and OVERWRITE. Select the mode deliberately: File export OVERWRITE replaces existing data while retaining the table and grants when its schema matches; +APPEND can duplicate outputs on retry. A column mismatch fails rather than silently changing the existing schema. Check destination +existence and access first. + +```python +import kumo.engine as kumo +from kumoapi.jobs import MetadataField, PredictionArtifactType, WriteMode + +# APPEND only; JOB_TIMESTAMP alone for a query with no time window. +RUN_COLUMNS = [MetadataField.JOB_TIMESTAMP, MetadataField.ANCHOR_TIMESTAMP] +output = kumo.PredictionOutputConfig( + connector=output_connector, + artifact_type=PredictionArtifactType.PREDICTIONS, + table_name=OUTPUT_TABLE, write_mode=WRITE_MODE, + extra_fields=RUN_COLUMNS if WRITE_MODE == WriteMode.APPEND else [], +) +written = trainer.predict( + population, snapshot=snapshot, output_config=output, **PREDICT_OPTIONS, +) +print(written.num_rows, written.files) +``` + +**APPEND rows say which run wrote them.** An APPEND write adds `JOB_TIMESTAMP` (when the run +started) and `ANCHOR_TIMESTAMP` (the time its predictions are for), so the table's history can be +ordered and filtered by run, and a retried run shows as a second `JOB_TIMESTAMP`. A query with no +time window has no anchor: pass `JOB_TIMESTAMP` alone, or the write fails after scoring. + +The write never adds a column to an existing table. When an APPEND destination already exists, +read its columns before generating the artifact. If it lacks either column, a run would fail at +the write, after training and scoring. Say so at the gate and let the user pick: add the missing +columns (`ALTER TABLE … ADD COLUMN`, type `TIMESTAMP_NTZ`; earlier rows hold nulls), a new table +the first run creates, or OVERWRITE. Adding columns is a write, confirmed like any other. + +The exact table name is used; no suffix is appended and its case is kept. Read back, with the +session taken in the first cell (`kumoai-reference/references/kumo_finetuning_sdk.md` §3.3), using +the quoted table name of §9.1 there, inspect schema and bounded values. For a new/replaced table compare its count +with `written.num_rows`. For APPEND compare the count delta against a recorded pre-write count +only without concurrent writers, or use a run-specific destination for unambiguous verification. +Check entity/anchor/class coverage too. A count alone does not prove semantic correctness. + +## Requested embeddings + +Predictions and embeddings are separate calls and destinations. For a fitted or loaded trainer, +using the same compatible snapshot and population: + +```python +embedding_output = kumo.PredictionOutputConfig( + connector=output_connector, + artifact_type=PredictionArtifactType.EMBEDDINGS, + table_name=EMBEDDING_TABLE, write_mode=WRITE_MODE, +) +embeddings = trainer.predict( + population, snapshot=snapshot, output_config=embedding_output, **PREDICT_OPTIONS, +) +print(embeddings.num_rows, embeddings.files) +``` + +Preserve task-specific arguments, including the explicit binary threshold where the API +requires it. Do not promise embeddings for every module/task: report an unsupported selection +error without silently switching models or refitting. Link models can expose entity and item +vectors; inspect TYPE and identifiers before calling them customer/product embeddings. + +## Serialization boundaries + +On Snowflake, the public prediction path exports selected Parquet through Snowflake's file sink, +with schema inference and COPY. This differs from the separate in-memory DataFrame writer, which rejects +list/dict columns. Do not apply that rejection indiscriminately to prediction export, and do +not send nested cohort JSON through the scalar DataFrame path unchanged. + +For vectors or other nested fields, inspect the destination's actual column type and a +bounded read-back value; inference is not a guarantee of the consumer's expected representation. +Use [stage retention](../../kumo-model-persistence/references/retention.md) to preserve the original +Parquet or JSON when a destination cannot represent the required shape. Conversion/flattening +requires an explicit schema decision. Do not claim that all artifact families fit an ordinary +table unchanged or automatically rerun prediction after an export failure. diff --git a/skills/kumo-data-science/kumoai-reference/references/pql_writing.md b/skills/kumo-data-science/kumoai-reference/references/pql_writing.md new file mode 100644 index 000000000..eabb16be2 --- /dev/null +++ b/skills/kumo-data-science/kumoai-reference/references/pql_writing.md @@ -0,0 +1,502 @@ +# PQL — Predictive Query Language + +Constructs marked ``Source: shipped package (``)`` are verified against the Kumo package source and are ground truth. The rest of this page is the PQL surface as previously compiled; it is **not** re-verifiable against the published Kumo documentation, which is stale for this version and is not a source here — do not cite it, fetch it, or reconcile against it. Anything explicitly not verified is marked with a `>` note. + +**Never emit a construct that is not on this page.** If the user's request needs one that is absent, say so rather than reaching for the docs or memory. + +PQL is the query language; the surrounding Python is in `kumoai-reference/references/graph_construction.md` and `kumoai-reference/references/kumo_finetuning_sdk.md`. + +**Never invent PQL syntax.** If the user's request cannot be expressed with the constructs on this page, say so and ask for a reformulation. + +**Scope rule:** KumoRFM is a different product with a different validator (`FOR customers.customer_id=42`, `RANK TOP K` capped at 20, no joined static filters). Do not mix it in. Platform/fine-tuning PQL uses `FOR EACH`. + +--- + +## Grammar + +```text +PREDICT +FOR EACH . +[WHERE ] +[ASSUMING ] +``` + +- `PREDICT` and `FOR EACH` are **required**. `WHERE` and `ASSUMING` are optional. +- `FOR EACH` accepts **only a primary key column**. The entity table must be directly connected to the target table by a PK→FK relationship. +- `RANK TOP ` / `CLASSIFY` are suffixes on the target expression, inside the `PREDICT` clause. + +> `RANK TOP ` has been seen both on its own line between `PREDICT` and `FOR EACH` and inline at the end of the `PREDICT` line. Prefer the inline form. + +### Target forms + +```text +PREDICT
. -- static / column target +PREDICT (
., , [, ]) -- temporal aggregation target +PREDICT -- conditional -> classification +PREDICT LIST_DISTINCT(
., , ) RANK TOP -- ranking / link prediction +PREDICT LIST_DISTINCT(
., , ) CLASSIFY -- per-class binary classification +``` + +--- + +## Aggregation functions + +Valid in the target and in temporal filters. + +| Function | Applies to | +|---|---| +| `SUM` | numerical columns | +| `AVG` | numerical columns | +| `MAX` | numerical, timestamp | +| `MIN` | numerical, timestamp | +| `COUNT` | all columns (use `TABLE.*` to count rows regardless of N/A) | +| `COUNT_DISTINCT` | all columns | +| `LIST_DISTINCT` | all columns; recommended only if fewer than ~1000 possible values | +| `LAST` | all columns **except foreign keys** | +| `FIRST` | all columns **except foreign keys** | + +Forecasting targets support **only** `SUM`, `COUNT`, `COUNT_DISTINCT` (`AVG` is undefined when an entity has no fact rows in a timeframe). + +### Null / empty-window behavior (critical, differs per function) + +- `SUM`, `COUNT`, `COUNT_DISTINCT`: if no fact rows exist in the window, the training label is **0**. +- `AVG`, `MIN`, `MAX`, `FIRST`, `LAST`: if no fact rows exist, the training example is **dropped**. This implicitly assumes every predicted entity has at least one fact row in the horizon. Use `IS NULL` / `IS NOT NULL` if you need to predict periods with no interactions. + +This is the difference between "will spend $0" (`SUM`/`COUNT`, keeps inactive entities) and "what will they buy next" (`FIRST`/`LAST`, silently drops inactive entities). + +--- + +## Time window / horizon + +The horizon is expressed **inside the aggregation function**, not as a separate clause. There is no `WITHIN` keyword. + +```text +(
., , , ) +``` + +- `start` is **exclusive**, `end` is **inclusive**. +- Both must be **non-negative integers**; `end` must be strictly greater than `start`. +- In the **target**, the window is forward-looking (positive). In a **temporal filter**, use negative offsets for the past. +- `unit` defaults to `days`. Valid: `minutes`, `hours`, `days`, `months` (uppercase also accepted). **Not** accepted: `seconds`, `weeks`, `years`, or any singular form (`day`, `week`). + +Worked example: predicting on `2020-01-01 00:00:00` with `SUM(TRANSACTIONS.PRICE, 10, 30)` aggregates rows with timestamp `t` where `2020-01-11 00:00:00 < t <= 2020-01-31 00:00:00`. + +```text +PREDICT COUNT(PURCHASES.*, 0, 48, hours) > 20 +PREDICT SUM(Purchases.amount, 0, 6, months) > 500 +``` + +Guidance: the fact table should hold history at least an order of magnitude longer than the prediction horizon (e.g. 1 year of rows for a 30-day horizon). At least 3 anchor times are needed for a healthy train/val/test split, so very large horizons can starve the splits. + +--- + +## Comparison operators + +Used to turn a target into a conditional (classification) target, and in filters. + +| Operator | Applies to | +|---|---| +| `>` `<` `>=` `<=` | numeric, timestamp | +| `=` `!=` | all column types | +| `STARTS WITH` | text | +| `ENDS WITH` | text | +| `CONTAINS` / `NOT CONTAINS` | text | +| `LIKE` / `NOT LIKE` | text, `%` wildcard | +| `IN` | e.g. `LOAN.STATUS IN ('A', 'C')` | +| `IS NULL` / `IS NOT NULL` | all column types (`= NULL` / `!= NULL` also shown) | + +Logical combinators: `AND`, `OR`, `NOT`. Each operand must itself be a filter operation. Supported in all filter fields and in targets. + +```text +LAST(LOAN.STATUS, 0, 30) CONTAINS 'SUCCESS' AND LAST(LOAN.AMOUNT, 0, 30) > 30 +NOT LAST(LOAN.AMOUNT, 0, 30) > 30 +ARTICLES.DESCRIPTION NOT CONTAINS 'polyester' +LAST(LOAN.STATUS, 0, 30) LIKE '%SUCCESS' +``` + +--- + +## Filters + +### `WHERE` — entity / temporal filter + +Restricts which entities (and therefore which training examples) are included. + +```text +PREDICT SUM(transactions.price, 0, 30) +FOR EACH customers.customer_id +WHERE COUNT(transactions.*, -30, 0) > 0 +``` + +```text +PREDICT fraud_reports.is_fraud +FOR EACH transaction.id +WHERE transaction.type = "bank transfer" +``` + +Temporal filters use negative offsets (look-back). Static filters compare a column to a constant. + +### Inline `WHERE` inside an aggregation + +Restricts the rows that the aggregation counts, rather than the entities. + +```text +PREDICT LIST_DISTINCT( + transactions.article_id + WHERE transactions.price > 100, +0, 30, days) +RANK TOP 25 +FOR EACH customers.customer_id +``` + +Note the syntax: the inline `WHERE` goes **after the column reference and before the `start, end, unit` arguments**. It may also be written on one line: + +```text +COUNT_DISTINCT(Ratings.rating WHERE Ratings.rating < 3, 0, 60, days) +``` + +Source: shipped package (`kumopql.grammar`) + +**Inline vs trailing `WHERE` is the same keyword doing two different jobs** — inline selects rows for the aggregation, trailing selects eligible entities. A single query may use both; see "Both `WHERE`s in one query" under the worked examples. + +### `ASSUMING` — hypothetical / future condition + +Followed by one or more future-looking temporal aggregations assumed true at prediction time. + +```text +PREDICT SUM(TRANSACTIONS.PRICE, 0, 30) +FOR EACH CUSTOMERS.CUSTOMER_ID +ASSUMING COUNT(NOTIFICATIONS.*, 0, 7) > 2 +``` + +Also usable as a way to drop zero-heavy training rows in forecasting: + +```text +PREDICT SUM(Sales.quantity, START_DAYS, END_DAYS) +FOR EACH Groups.group_id +ASSUMING SUM(Sales.quantity, START_DAYS, END_DAYS) > 0 +``` + +`ASSUMING` is not a causal estimate. Do not present a difference between assumed and unassumed predictions as proof of causation. + +--- + +## Date and time literals in filters (gotcha) + +Timestamp constants are written as **bare ISO 8601, unquoted** — no quotes, no `DATE '...'` wrapper, no `TIMESTAMP` cast. Hours/minutes/seconds may be included and default to `0`. + +```text +PREDICT COUNT(transactions.*, 0, 10) +FOR EACH customers.customer_id +WHERE customers.date_joined < 2022-01-01 12:45:30 +``` + +```text +PREDICT CUSTOMERS.industry = 'IT' AND CUSTOMERS.date_of_birth <= 1990-01-01 +``` + +String literals in filters appear both single-quoted (`'bank transfer'`, `'IT'`) and double-quoted (`"bank transfer"`). + +--- + +## Task type is inferred from query shape + +Kumo determines the task type automatically. `pquery.get_task_type()` returns it — always call it to confirm intent. + +| Query shape | Task type | +|---|---| +| `PREDICT
.` | Regression (static) | +| `PREDICT SUM/AVG/MIN/MAX/COUNT(...)` with no comparison | Regression (temporal) | +| Any target with a conditional operator (`=`, `>`, `<`, …) | Binary classification | +| `PREDICT FIRST/LAST(, …)` over a high-cardinality categorical | Multiclass classification | +| `LIST_DISTINCT` over a **non-foreign-key** categorical column | Multilabel classification | +| `LIST_DISTINCT` over a **foreign key** column (+ `RANK TOP K`) | Link prediction / ranking | +| `LIST_DISTINCT(...) CLASSIFY` | Separate binary classification per class | +| Temporal regression + model-plan `forecast_length` | Forecasting | + +Static vs temporal: a query without a time window is **static** (default 80/10/10 random split, no time column required). A query aggregating over a window is **temporal** (time-based splits, anchor times sampled back through history). + +--- + +## Worked examples by task type + +### Binary classification — churn + +```text +PREDICT COUNT(transactions.*, 0, 90, days) = 0 +FOR EACH customers.customer_id +WHERE COUNT(transactions.*, -60, 0, days) > 0 +``` + +Predicts whether a customer active in the last 60 days makes zero transactions in the next 90. `=` makes it classification; swapping to `>` changes which class is positive. Output is the probability of the positive class. + +### Binary classification — will a *qualifying* event occur (temporal, filtered) + +Use this shape for "will entity X have at least one event of a particular kind in the next N days" — fraud, conversion, unsubscribe, churn-by-event. The condition that defines *which* rows count goes **inside** the aggregation. + +```text +PREDICT COUNT(transactions.* WHERE transactions.is_fraud = 1, 0, 30, days) > 0 +FOR EACH accounts.account_id +``` + +When the label is already a numeric 0/1 flag, summing it is equivalent: + +```text +PREDICT SUM(fraud_reports.LABEL, 0, 30, days) > 0 +FOR EACH accounts.account_id +``` + +**Do not** express the event condition as an entity-level `WHERE`. These mean different things and only the first is correct here: + +| Query | Meaning | +|---|---| +| `PREDICT COUNT(t.* WHERE t.is_fraud = 1, 0, 30, days) > 0` | will this account have a **fraudulent** transaction | +| `PREDICT COUNT(t.*, 0, 30, days) > 0 ... WHERE t.is_fraud = 1` | will this account have **any** transaction, training only on accounts that already had fraud | + +The second leaks the label into entity selection and predicts the wrong thing. If the target condition describes the *future event*, it belongs inline; if it describes *which entities are eligible*, it belongs in the trailing `WHERE`. + +Also do not reach for the static form (`PREDICT fraud_reports.is_fraud`) when the request names a time window — static queries have no horizon and cannot answer "in the next 30 days". + +#### Both `WHERE`s in one query — the canonical example + +Source: shipped package (`kumopql.grammar`, `kumo.engine.pquery.predictive_query`) + +This query parses under the PQL grammar (`PredictiveQuery.validate()` parses and validates the query string against the graph) and uses **both** jobs of the `WHERE` keyword at once. Copy this shape when a request has a row-level target condition *and* an entity-level cohort condition. + +```text +PREDICT COUNT_DISTINCT(Ratings.rating WHERE Ratings.rating < 3, 0, 60, days) > 0 +FOR EACH Movies.MovieID +WHERE AVG(Ratings.rating, -60, 0, days) > 3 +``` + +Read it clause by clause: + +| Clause | Job | +|---|---| +| `WHERE Ratings.rating < 3` (inline, inside `COUNT_DISTINCT`) | selects **which rows** the aggregation counts — only low ratings | +| `0, 60, days` | forward target window | +| `> 0` | makes it binary classification: at least one low rating occurs | +| `WHERE AVG(Ratings.rating, -60, 0, days) > 3` (trailing) | selects **which entities** are eligible — only movies well rated over the past 60 days | + +Same keyword, two different jobs, in one query: "will a currently well-rated movie start getting low ratings?". Moving the inline condition to the trailing `WHERE` would instead train only on movies that already had low ratings and predict whether they get *any* rating — a different and wrong question. + +Note also the offsets: the target window is non-negative (`0, 60`), the entity filter looks back (`-60, 0`). + +### Binary classification — static / non-temporal + +```text +PREDICT fraud_reports.is_fraud +FOR EACH transaction.id +WHERE transaction.type = "bank transfer" +``` + +```text +PREDICT customer_item_pairs.likes = 1 +FOR EACH customer_item_pairs.uuid +``` + +Rows where the label is NULL are dropped from training and become the prediction set. + +### Multiclass classification — next purchase category + +```text +PREDICT FIRST(purchase.type, 0, 7) +FOR EACH user.user_id +``` + +`FIRST` is undefined when a user made no purchase in the window, so those examples are dropped — every prediction assumes the user buys something. + +### Regression — static + +```text +PREDICT customers.age +FOR EACH customers.customer_id +``` + +### Regression — temporal (LTV) + +```text +PREDICT SUM(transactions.price, 0, 30) +FOR EACH customers.customer_id +WHERE COUNT(transactions.*, -30, 0) > 0 +``` + +### Temporal link prediction / recommendation + +```text +PREDICT LIST_DISTINCT(transactions.article_id, 0, 30, days) +RANK TOP 25 +FOR EACH customers.customer_id +``` + +```text +PREDICT LIST_DISTINCT(transactions.article_id, 0, 7) RANK TOP 10 +FOR EACH customers.customer_id +``` + +With an inline filter on candidate items: + +```text +PREDICT LIST_DISTINCT( + transactions.article_id + WHERE transactions.price > 100, +0, 30, days) +RANK TOP 25 +FOR EACH customers.customer_id +``` + +Output is K rows per entity by default; `num_classes_to_return` at prediction time can return a different count (1–500 for link prediction; `kumoai-reference/references/kumo_finetuning_sdk.md` §9). Ranking tasks have a limit of 10 million target items. `RANK` on a foreign key is the only case where new targets may be added at batch-prediction time (by adding rows to the item table). + +### Static link prediction — item-to-item similarity + +```text +PREDICT LIST_DISTINCT(copurchase.dst_item_id) RANK TOP 10 +FOR EACH src_item.item_id +``` + +`LIST_DISTINCT` with **no window arguments** → static link prediction. + +### Multilabel via CLASSIFY + +```text +PREDICT LIST_DISTINCT(TRANSACTIONS.ARTICLE_ID, 0, 30) CLASSIFY +FOR EACH CUSTOMERS.CUSTOMER_ID +``` + +Produces a separate binary classification per article ID. Without `TOP K`, classes are generated for all available classes up to the limit. + +### Forecasting + +Forecasting is **not a distinct PQL keyword**. Write a temporal regression query and set `forecast_length` in the model plan; `forecast_length=3` on a 7-day window forecasts the next three 7-day windows. + +```text +PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 7, days) +FOR EACH CUSTOMERS.CUSTOMER_ID +``` + +```text +PREDICT COUNT(Sales.*, 0, 28) +FOR EACH Items.item_id +``` + +```text +-- 6th week out +PREDICT SUM(Sales.quantity, 35, 42) +FOR EACH Groups.group_id +``` + +Targets restricted to `SUM`, `COUNT`, `COUNT_DISTINCT`. `forecast_type` (`mlp` default, `implicit`, `autoregressive`) is a model-plan option. Treat forecasting as **experimental**. + +--- + +## Evaluation metrics per task type + +| Task type | Metrics Kumo reports | +|---|---| +| Binary classification | Accuracy, AUROC, AUPRC, confusion matrix, gain chart | +| Multiclass classification | Accuracy | +| Multilabel classification | AUPRC, AUROC, AP (macro / micro / per-label); also F1, Hamming score | +| Link prediction | F1@K, MAP@K, Precision@K, Recall@K for K = 1, 10, 100 | +| Regression | MAE, MSE, RMSE, MAPE, SMAPE, prediction-distribution histogram | + +Additional selectable metrics (model-plan `metrics` field): binary adds `ap`, `f1`, `ndcg[@k]`, `precision[@k]`, `recall[@k]`; multiclass adds `f1`, `precision`, `recall`; link prediction adds `mrr@k`, `hit_ratio@k`, `coverage@k`, `avg_popularity@k`, `personalization@k`, `diversity[col]@k`; forecasting adds `neg_binomial`, `normal`, `lognormal`. + +**Defaults:** the SDK's default model plan sets `metrics` and `tune_metric` per task type, for +every run mode: + +| Task type | Reported metrics | Tuned on | +|---|---|---| +| Binary classification | `auroc`, `ap`, `auprc` | `auroc` | +| Multiclass classification | `acc` | `acc` | +| Multilabel classification | `ap`, `auroc`, `auprc`, each `_macro` / `_micro` / `_per_label` | `ap_macro` | +| Regression, forecasting | `mae`, `mse`, `rmse` | `mae` | +| Ranking, link prediction | `map`, `precision`, `recall`, `f1`, each `@1` / `@10` / `@100` | `map@10` | +| Link prediction with a training-table weight column | `weighted_ndcg`, `weighted_recall`, `precision`, each `@1` / `@10` / `@100` | `weighted_ndcg@10` | + +Source: shipped package (`kumo_core.model_plan`) + +--- + +## Gotchas and malformed-query traps + +1. **No `WITHIN` / `ON` / `TOP_K` / `NEXT_ITEM` keywords.** The horizon lives in the aggregation arguments; the entity clause is `FOR EACH`; ranking is `RANK TOP K`. +2. **Do not mix static conditions and temporal aggregations in one target.** Temporal aggregations depend on anchor time, static conditions do not; they need different sampling strategies. Documented as invalid practice. +3. **`start` must be non-negative and `end` strictly greater than `start`** in the target. Negative offsets only in temporal filters. +4. **Unit names must be plural and from the allowed set.** `day`, `week`, `weeks`, `years`, `seconds` are rejected. +5. **Date literals are unquoted ISO 8601.** See the date/time section. +6. **`FOR EACH` requires a primary key.** If the entity table has duplicate rows, pre-build a `SELECT DISTINCT` view first. +7. **FK direction matters.** The entity table must connect to the target table via PK→FK. In the fraud example, `transaction` holds the FK to `fraud_reports`, not the reverse — otherwise multiple reports could link to one transaction. +8. **`LAST`/`FIRST` cannot be applied to foreign key columns.** Use `LIST_DISTINCT` + `RANK` for FK targets. +9. **`LIST_DISTINCT` over a non-FK column silently becomes multilabel classification, not ranking.** Check `get_task_type()`. +10. **`LIST_DISTINCT` beyond ~1000 distinct values is not recommended.** +11. **Horizon vs. split health:** too large a window leaves fewer than 3 anchor times; too small a window on a long history generates too many examples (mitigate with model-plan `train_start_offset`) and, for link prediction, too few positives. +12. **Always validate before generating tables:** `pquery.validate()` for syntax, `pquery.get_task_type()` for semantics. Required — see "Required pre-flight" under SDK usage. +13. **Inline `WHERE` vs trailing `WHERE` are not interchangeable.** A row-level condition on the predicted event goes inside the aggregation; an entity-eligibility condition goes in the trailing `WHERE`. Swapping them produces a query that validates and trains on the wrong question. + +--- + +## SDK usage + +`PredictiveQuery(graph=graph, query=)` binds the graph at construction. Call `validate()` and `get_task_type()` with no arguments. + +```python +import kumo.engine as kumo + +pquery = kumo.PredictiveQuery( + graph=graph, + query=( + "PREDICT MAX(transaction.Quantity, 0, 30, days)\n" + "FOR EACH customer.CustomerID\n" + "ASSUMING SUM(transaction.UnitPrice, 0, 7, days) > 15" + ), +) + +pquery.validate() +print(pquery.get_task_type()) +``` + +Build multi-line query strings with a triple-quoted string and `.strip()`. Explicit `\n` separators also work. + +```python +QUERY = """ +PREDICT COUNT_DISTINCT(Ratings.rating WHERE Ratings.rating < 3, 0, 60, days) > 0 +FOR EACH Movies.MovieID +WHERE AVG(Ratings.rating, -60, 0, days) > 3 +""".strip() +``` + +Table names used in PQL are the **source table names** — `Graph(tables=[...])` has no keys. See `kumoai-reference/references/graph_construction.md`. + +Source: shipped package (`kumo.engine.pquery.predictive_query`) + +### Required pre-flight, before any training step + +Source: shipped package (`kumo.engine.pquery.predictive_query`) + +Always emit these two calls, in this order, immediately after constructing the `PredictiveQuery` and before generating training tables or fitting: + +```python +pquery.validate() # syntax + schema; call with no arguments +print('task type:', pquery.get_task_type()) +``` + +`validate()` catches syntax and graph-mismatch errors; `get_task_type()` is the semantic check: the task type is never written in the query, only inferred from its shape, so print it and confirm it matches the user's intent before spending a training run. A query that validates can still be the wrong task (see the `LIST_DISTINCT` multilabel-vs-ranking trap below). + +> There is no `KumoFineTuning` class. Do not emit `kumo.init()` — hosted-edition only, out of scope. + +> `RANK BY` appears in one stray example elsewhere but is defined nowhere and contradicts the `LIST_DISTINCT(...) RANK TOP K` form used here. Treat `RANK BY` as unverified; **do not emit it.** + +--- + +## Mapping a validated request to PQL + +| Validated field | PQL element | +|---|---| +| Prediction target | `PREDICT` expression — pick the aggregation from the target column type | +| Primary entity | `FOR EACH .` | +| Prediction horizon | `start`, `end`, `unit` arguments inside the aggregation | +| Eligibility / cohort constraint (which **entities**) | trailing `WHERE` (temporal look-back or static column filter) | +| Condition on the predicted event (which **rows** count) | inline `WHERE` inside the aggregation, before the window arguments | +| Hypothetical future condition | `ASSUMING` | +| Number of recommendations | `RANK TOP ` | +| Task type | Not written explicitly — inferred from query shape; confirm with `get_task_type()` | diff --git a/skills/kumo-data-science/kumoai-reference/references/task_journeys.md b/skills/kumo-data-science/kumoai-reference/references/task_journeys.md new file mode 100644 index 000000000..5d888d736 --- /dev/null +++ b/skills/kumo-data-science/kumoai-reference/references/task_journeys.md @@ -0,0 +1,111 @@ +# Complete a task journey + +Start with a confirmed graph, query, source scope, population, evaluation protocol and workload. +The examples below share the [SDK setup](kumo_finetuning_sdk.md). Symbols in uppercase are +user-selected inputs; table/column names in PQL are illustrative and must match the actual +graph. Each journey validates the query, generates populations, suggests a model plan, fits, +predicts and inspects output. Do not execute every example as one notebook. + +## Classification and regression + +Choose one query from [PQL forms](pql_writing.md), then set `EXPECTED_TASK` to its confirmed +public `TaskType`, imported from `kumoapi.task`. `IS_TEMPORAL` comes from the confirmed query; +`SCORING_TIME` is the requested naive datetime in UTC. `PREDICT_OPTIONS` contains only supported +options for that task, as listed below. + +```python +import kumo.engine as kumo +from kumoapi.model_plan import RunMode + +pq = kumo.PredictiveQuery(graph=graph, query=QUERY) +pq.validate() +if pq.get_task_type() != EXPECTED_TASK: + raise ValueError("Resolved query task differs from confirmed intent") +snapshot = graph.snapshot() +training_table = pq.generate_training_table(snapshot) +plan = pq.suggest_model_plan(snapshot, run_mode=RunMode.FAST) +# Apply only previously requested workload/configuration choices here. +trainer = kumo.Trainer(plan) +training_result = trainer.fit(training_table, snapshot=snapshot) +anchors = {"prediction_time": SCORING_TIME} if IS_TEMPORAL else {} +population = pq.generate_prediction_table(snapshot, **anchors) +predictions = trainer.predict(population, snapshot=snapshot, **PREDICT_OPTIONS) +print(training_result.metrics) +print(predictions.num_rows, predictions.files) +``` + +| Journey | Task confirmation, scoring options and interpretation | +| --- | --- | +| Static or temporal binary | Confirm BINARY_CLASSIFICATION and positive condition. Supply `binary_classification_threshold=THRESHOLD` in [0,1]. `True_PROB`/`False_PROB` are probabilities and `TARGET_PRED` is the thresholded label. Compare AUROC/AP and threshold metrics only when recorded | +| Static or temporal multiclass | Confirm MULTICLASS_CLASSIFICATION and class meaning. No binary threshold. Optional `num_classes_to_return` limits returned classes only when requested. Long rows carry CLASS, SCORE and PREDICTED with entity metadata. Compare compatible class/averaging metrics and per-class cohorts when present | +| Static or temporal multilabel | Confirm MULTILABEL_CLASSIFICATION (or explicitly intended MULTILABEL_RANKING) and label set. No explicit binary threshold for multilabel prediction output: the API rejects it. Long CLASS/SCORE/PREDICTED rows represent separate label outcomes; do not interpret scores as exclusive class probabilities | +| Static or temporal regression | Confirm REGRESSION and units. No threshold or top-class count. Inspect TARGET_PRED, MAE/RMSE when recorded, and matching temporal/split semantics | + +Static tasks omit prediction_time entirely, and a static query scores only rows whose target is +NULL; to score rows that already have one, use the population table confirmed at the task gate +(`kumoai-reference/references/kumo_finetuning_sdk.md` §7.1). Supplied exact populations use +[custom populations](../../kumo-custom-populations/SKILL.md) instead of generated populations; +never attach evaluation labels to a scoring table. For generated static tasks, verify which +rows are eligible before claiming all entities were scored. + +## Links and ranking + +Use the same journey with a confirmed FK target, for example +`PREDICT LIST_DISTINCT(orders.product_id, 0, 30, days) RANK TOP 10 FOR EACH customers.customer_id`. +For static links use `LIST_DISTINCT(pairs.product_id)` without a time window and omit the +scoring timestamp. Confirm TEMPORAL_LINK_PREDICTION or STATIC_LINK_PREDICTION respectively. +Set `PREDICT_OPTIONS = {"num_classes_to_return": N}`, where N is the confirmed number of +results per entity: the query's `RANK TOP K` unless the user asked for a different count +(1–500; `kumoai-reference/references/kumo_finetuning_sdk.md` §9). No binary threshold. +Preserve the item universe, positive interaction, eligibility and evaluation sampling. Inspect +entity/CLASS and available SCORE/time columns. Returned rows can be fewer than N when +candidates are unavailable. Compare recorded precision/MAP/recall at the same K. + +For a non-FK category ranking, confirm the distinct MULTILABEL_RANKING task rather than treating +it as a link to a product table. Do not create a graph edge solely to force a task type. + +## Forecasting + +Confirm interval, number of steps, target units, required history and evaluation horizon. +Use SUM, COUNT or COUNT_DISTINCT over that interval. The initial query can resolve as +REGRESSION before generation settings promote it to forecasting; use the generation plan in this workflow, not a `ModelPlan.training_table_generation` field. + +This example forecasts `FORECAST_STEPS` consecutive windows for the confirmed temporal query: + +```python +from kumoapi.model_plan import TrainingTableGenerationPlan, RunMode +from kumoapi.task import TaskType + +pq = kumo.PredictiveQuery(graph=graph, query=FORECAST_QUERY) +pq.validate() +if pq.get_task_type() not in (TaskType.REGRESSION, TaskType.FORECASTING): + raise ValueError("Forecast query must have a supported numerical temporal target") +snapshot = graph.snapshot() +generation = TrainingTableGenerationPlan(forecast_length=FORECAST_STEPS) +training_table = pq.generate_training_table(snapshot, plan=generation) +plan = pq.suggest_model_plan(snapshot, run_mode=RunMode.FAST) +trainer = kumo.Trainer(plan) +training_result = trainer.fit(training_table, snapshot=snapshot) +population = pq.generate_prediction_table( + snapshot, prediction_time=SCORING_TIME, + plan=trainer.training_table_generation_plan, +) +predictions = trainer.predict(population, snapshot=snapshot) +print(trainer.training_table_generation_plan) +print(training_result.metrics, predictions.num_rows, predictions.files) +``` + +Inspect effective generation metadata to confirm the intended forecast length, lags and +intervals. Prediction must reuse the fitted generation settings. Outputs expand into one row +per entity/forecast timestamp with TARGET_PRED and ANCHOR_TIMESTAMP; configured quantiles can +add q_* columns. Compare horizon-aligned errors and temporal splits, not one-window regression +scores as if they established all horizons. Column analysis is not supported for forecasting. + +## Finish each journey + +Use [outputs](outputs.md) for requested write-back and a separate embedding call. +Inspect selected Parquet schemas before interpreting columns; do not hardcode optional fields. +Use [diagnostics](../../kumo-training-diagnostics/SKILL.md) for actual metrics and trial histories, +and [explanations](../../kumo-model-explanations/SKILL.md) for available attribution/cohorts. +Missing explanations do not imply training failed. Retain requested artifacts via +[persistence](../../kumo-model-persistence/SKILL.md) before notebook teardown. diff --git a/skills/kumo-data-science/kumoai-reference/references/troubleshooting_signals.md b/skills/kumo-data-science/kumoai-reference/references/troubleshooting_signals.md new file mode 100644 index 000000000..b497ce5c9 --- /dev/null +++ b/skills/kumo-data-science/kumoai-reference/references/troubleshooting_signals.md @@ -0,0 +1,274 @@ +# Troubleshooting Signals + +Observable symptom → likely cause → fix. Ordered by when the agent hits them: table definition, graph construction, query validation, then job time. + +This file is the signal set as previously compiled. It is **not** re-verifiable against the published Kumo documentation, which is stale for this version of the local SDK and is not a source here — do not cite it, fetch it, or reconcile against it. + +> No verbatim Kumo exception text is established for this SDK. Entries below are keyed on the **observable condition**, not on a quoted message. **Do not fabricate Kumo error strings.** + +--- + +## 0. The validation calls — run these, do not guess + +These exist in the current package. **Call every one with no arguments.** + +| Call | Checks | +|---|---| +| `table.infer_metadata()` | Re-resolves `dtype` / `stype` from the source | +| `graph.validate()` | Graph structure, before any snapshot. Cheap — schemas only | +| `pquery.validate()` | Query syntax and its fit against the graph | +| `pquery.get_task_type()` | The inferred task type — confirms the ML problem matches intent | + +Standard order: build tables -> `graph.validate()` -> `pquery.validate()` -> +`get_task_type()` -> `graph.snapshot()` -> generate tables -> train. + +**These do not exist.** Do not emit them, and do not tell the user to run them: +`table.validate()`, `table.metadata`, `table.get_stats()`, `graph.infer_links()`, +`graph.get_edge_stats()`, `graph.get_table_stats()`, `connector.sql(...)`, +`SourceTable.head(...)`. + +Inspect a table through `.definition` (`.pkey`, `.time_col`, `.end_time_col`) and a graph through +`.tables` / `.edges`. To look at data, read the parquet that `snapshot()` and +`generate_training_table()` leave under the scratch directory. + +--- + +## 1. Table definition + +### Symptom: `connector.table("X")` raises `ValueError` +**Cause:** The table name does not exist in the connector. +**Fix:** List what is actually there before constructing tables: `connector.table_names()`, or guard with `connector.has_table(name)`. Snowflake object names, for example, are typically uppercase (`connector.table("ORDERS")`). + +### Symptom: a column's `stype` is unset, or the query fails because a column has no semantic type +**Cause:** `Table.from_source_table()` and the explicit `Table(...)` constructor do **not** fully populate `dtype`/`stype`. A partially specified table is legal until it is used in a predictive query. +**Fix:** Call `.infer_metadata()`, or set the type explicitly: +```python +table.column("col").stype = "categorical" +``` +Fully specify tables in production code to avoid unexpected re-inference. + +### Symptom: "invalid data type error" / type-mismatch recommendation on a column +**Cause:** The semantic type (`Type`) does not align with the data type (`Data Type`). +**Fix:** Align them. Valid `Stype` values: `numerical`, `categorical`, `multicategorical`, `ID`, `text`, `timestamp`, `sequence`, `image`, `unsupported`. + +### Symptom: a column holds URLs, lat/long, IP addresses, or phone numbers +**Cause:** These types are **not supported** for preprocessing. +**Fix:** Transform before ingestion — extract domain/path, geographic area, subnet, or area code and treat as categorical. + +### Symptom: a column holds nested schemas, arrays, or maps +**Cause:** Not supported. +**Fix:** Flatten to a delimited string. `["TV", "electronics", "promotion"]` → `"TV, electronics, promotion"` (multicategorical). + +### Symptom: timestamps arrive as strings or are misparsed +**Cause:** Non-ISO-8601 format, or Parquet columns not cast to a `DATE`/`TIME`/`TIMESTAMP` type. +**Fix:** Use ISO 8601 or epoch time; cast Parquet timestamp columns at the source. + +### Symptom: row count silently drops after ingestion +**Cause:** Duplicate primary key values — Kumo retains one row per key and drops the rest. +**Fix:** Deduplicate at the source, or pick a genuinely unique key. + +### Symptom: a categorical column has very high cardinality +**Cause:** Categorical supports up to **4,000 unique values by default**. +**Fix:** Bucket the values, or re-type the column (`ID` for identifiers, `text` for multi-token strings). + +### Symptom: missing values are filled with `N/A`, `none`, `null`, or `-1` +**Cause:** These placeholders are interpreted as real values. +**Fix:** Leave the field blank. A high-missingness column can still be useful when absence itself is informative. + +--- + +## 2. Graph construction (highest priority — fail here, not at job time) + +### Symptom: a table cannot be linked into the graph at all +**Cause:** To connect tables into a graph, **at least one table must contain a primary key**. Edges map a `src_table`'s foreign key to a `dst_table`'s primary key. +**Fix:** Set a primary key on the entity/dimension table: +```python +table.primary_key = 'new_primary_key' # must already be a column of the table +print(table.has_primary_key()) +``` + +### Symptom: an edge is declared but the model learns nothing from it +**Cause:** FK values do not actually match PK values. Rows with missing or unmatched key values are +**silently ignored** when linkages are built, so a broken join looks like a sparse one. If the two +linked columns have different types, both are **cast to strings** — values equal as numbers may not +match as strings (`1.0` vs `1`). +**Fix:** There is **no `get_edge_stats()` in this SDK.** Check the join yourself in SQL before +building the graph — count matching keys across the two tables — and normalize the key dtypes on +both sides. Then set every foreign key to `Stype.ID`; left as numerical it is treated as a feature +rather than a link. + +### Symptom: the edge direction looks wrong / no neighbors are sampled +**Cause:** `src_table` must be the table holding the **foreign key**; `dst_table` holds the matching **primary key**. Edges are bidirectional once declared, but the declaration itself is directional. +**Fix:** +```python +edges=[ + dict(src_table='transaction', fkey='StockCode', dst_table='stock'), + dict(src_table='transaction', fkey='CustomerID', dst_table='customer'), +] +``` + +### Symptom: graph exceeds known limits +**Cause:** Default quotas: 12 tables per graph, 15B rows per table, 2,048 columns per table, 2 TB per table, 30B rows per graph, 2 TB per graph, 30B edges. Environments may be configured lower. +**Fix:** Reduce table count or prune columns. Limits are admin-configurable; a limit increase requires the Kumo point of contact. + +### Symptom: an event table has no time column +**Cause:** Kumo uses time columns to enforce the anchor-time boundary. For event tables (each row a discrete occurrence), the time column should be when the event occurred or when its information became available. +**Fix:** Set `time_column` on the event table: +```python +transaction = kumo.Table.from_source_table( + source_table=transaction_src, + time_column='InvoiceDate', +).infer_metadata() +``` +Entity tables may use the entity creation time. Tables with no meaningful timeline can remain without a time column — that is legal. Do **not** designate an administrative "last modified" timestamp unless it represents when the row became valid for modeling. + +### Symptom: the user supplies pre-aggregated features (`logins_last_30_days`, etc.) +**Cause:** Aggregating before ingestion removes the sequence/relationship information the GNN learns from. Kumo reports aggregated features can perform **5–10% worse** than an equivalent GBDT model, while raw data performs better. +**Fix:** Ask for the raw timestamped event table instead. Do not do feature engineering on Kumo's behalf. + +### Symptom: the user has downsampled tables before ingestion +**Cause:** Independently sampling rows from each table breaks FK coverage and leaves entities without their histories. +**Fix:** Sample around the prediction entity and keep all related rows across every table — e.g. "all rows in all tables where `user_id % 10 == 1`". Never sample each table independently. + +--- + +## 3. Predictive query validation + +### Symptom: `pquery.validate()` reports a syntax error +**Cause:** PQL is `PREDICT FOR EACH ` with an optional filter clause. The query must reference the **names of tables as registered in the graph's `tables` dict**, not the source table names. +**Fix:** Re-check table names against `graph.tables`. Minimal valid form: +```python +pquery = kumo.PredictiveQuery( + graph=graph, + query=( + "PREDICT MAX(transaction.Quantity, 0, 30, days)\n" + "FOR EACH customer.CustomerID\n" + "ASSUMING SUM(transaction.UnitPrice, 0, 7, days) > 15" + ), +) +pquery.validate() +``` + +### Symptom: aggregation window rejected +**Cause:** Both *start* and *end* must be **non-negative integers**, and *end* must be **greater than** *start*. +**Fix:** Correct the bounds. Semantics: start is exclusive, end is inclusive. `SUM(TRANSACTIONS.PRICE, 10, 30)` from a `2020-01-01` anchor aggregates `2020-01-11 00:00:00 < t <= 2020-01-31 00:00:00`. + +### Symptom: the user asks for a horizon in an unsupported unit +**Cause:** The accepted time units are `days` (default), `months`, and `hours`. +**Fix:** Convert. Do not emit `weeks`, `quarters`, or `years` — they are not accepted. + +> `weeks` is not an accepted unit. Convert to `days`. + +### Symptom: `get_task_type()` returns a different task than intended +**Cause:** The PQL formulation determines the ML task, not the user's prose. The same business problem has several valid formulations. +**Fix:** Always call `get_task_type()` after `validate()` and confirm it against the user's intent before generating a training table. Reformulate the query if it disagrees. + +### Symptom: unclear whether the generated training data is what the user meant +**Cause:** Validation checks syntax, not semantics. +**Fix:** Inspect the generated data before training: +```python +training_table = pquery.generate_training_table(non_blocking=False) +print(training_table.data_df().head()) +``` +`data_df()` and `data_urls()` both expose the generated data. + +### Symptom: an `END_DATE` column is set on a non-entity table and appears to be ignored +**Cause:** End-date-aware sampling is currently supported **only** on the entity table (after `FOR EACH`) and on the RHS table in link prediction (after `LIST_DISTINCT`). +**Fix:** Do not rely on end dates elsewhere. + +### Symptom: date-only boundary includes one day too few / too many +**Cause:** A date such as `2025-10-01` is interpreted at the **start** of that day. The target window is left-open, right-closed. +**Fix:** To include all of October 1, use `2025-09-30 23:59:59` rather than midnight on October 1. + +--- + +## 4. Data leakage + +### Symptom: evaluation results are unexpectedly strong, offline and production performance diverge, or one column dominates explanations +**Possible cause:** Data leakage; these symptoms alone do not establish it. Check for: +- A feature directly or indirectly contains the future target. +- An event timestamp marks the start of a period while another column summarizes what happened by the end of it. +- A supposedly static entity column is overwritten over time, so its historical value is unavailable. +- Training and holdout periods overlap or are ordered incorrectly. + +**Fix:** Verify when each source value became available relative to the anchor time before +correcting the timeline or removing a confirmed leaking field. **Do not tune the model to +compensate for temporally invalid data.** For interpretation limits of recorded metrics and +explanations, follow the existing-run analysis route in `kumo-data-science/SKILL.md`. + +### Pre-flight data quality checklist +Run before building the graph: +- [ ] A source table contains the label needed for positive and negative training examples. +- [ ] Enough history to learn seasonal patterns — typically 1–2 years, sometimes more. +- [ ] The data fits within the quotas and limits above. +- [ ] PII is hashed, obfuscated, or removed per policy. +- [ ] No target leakage — columns mutated/updated over time must carry the timestamp at which the value first became known. +- [ ] The dataset contains the signals a human would expect to be predictive. +- [ ] Tables can all be linked through stable primary/foreign keys. + +Column-selection criteria: **clean** (no synthetic data, no other models' predictions, no columns whose definition changed over time), **relevant and mutually exclusive** (drop duplicate/irrelevant columns — graph size drives compute cost), **complete** (full history, not just first/last interaction). + +--- + +## 5. Training and prediction time + +### Symptom: `ValueError: prediction_time must not include a timezone` +**Cause:** `prediction_time` was passed as a timezone-aware `datetime`. `kumo_core` rejects any +`tzinfo` and applies UTC itself. +**Fix:** Pass a **naive** datetime. +```python +PREDICTION_TIME = datetime(2026, 8, 1) # correct +# PREDICTION_TIME = datetime(2026, 8, 1, tzinfo=timezone.utc) # raises +``` + +### Symptom: `ValueError: prediction_time is not supported for static queries` +**Cause:** `prediction_time` was passed for a non-temporal query. +**Fix:** Drop the argument. Confirm the intended task with `pquery.get_task_type()`. + +### Symptom: `fit()` or `predict()` raises `TypeError` about a missing argument +**Cause:** Both require `snapshot=` as a keyword. `fit(training_table)` and +`predict(prediction_table)` alone are errors. +**Fix:** +```python +training_result = trainer.fit(training_table, snapshot=snapshot) +prediction_result = trainer.predict( + prediction_table, snapshot=snapshot, output_config=output_config) +``` + +### Symptom: code from an older notebook raises `AttributeError` on `kumo.` +**Cause:** It targets the previous SDK — `set_backend`, `LocalBackend`, `create_identity`, +`OutputConfig`, `kumo.Stype`, edges written as dicts. A `Graph(tables={alias: table})` mapping +is current, not old. +**Fix:** Do not patch it call by call. Say it targets a superseded build and rewrite against +`kumoai-reference/references/kumo_finetuning_sdk.md` §10. The recognise-and-refuse table is §11.1 there. + +### Symptom: `snapshot()` warns ` invalid values found in
.` +**Cause:** Values in that column did not parse as its declared type. They are read as null; the +snapshot still completes. +**Fix:** Report the table, column and count to the user. A large count usually means a wrong +`dtype` (a TEXT date without `dtype=Dtype.date`, or a numeric column with string placeholders); +correct the declaration or the source and rebuild. Do not ignore it silently. + +### Symptom: `snapshot()` warns the scratch directory is on network storage +**Cause:** The scratch root is on network-backed storage such as EBS. Sampling and training-table +generation run much slower there. +**Fix:** It is a warning, not an error — a small run can continue. For a large run, point +`kumo.configure(scratch_dir=...)` at local NVMe if the runtime has it. + +### Symptom: link prediction job fails with an explicit out-of-memory error +**Cause:** Link prediction has **no** batch-size fallback. Node prediction, classification, regression, and forecasting catch OOM, halve the batch size, and retry; link prediction does not. +**Fix:** Reduce `output_embedding_dim` — the first remedy to try. + +### Symptom: results change between two training runs on the same query +**Cause:** The source data moved underneath the graph. +**Fix:** `graph.snapshot()` locks in the current version of all tables so repeated runs use the same data. It also reduces resource usage across multiple jobs. + +--- + +## 6. Known gaps + +> Not verified: +> - Verbatim exception text and error codes raised by `Graph.validate()` and +> `PredictiveQuery.validate()`. **Do not fabricate them.** +> - The exception types raised by graph or PQL validation failures. +> - Whether `pquery.validate()` catches nonexistent column references, or only syntax. diff --git a/skills/kumo-data-science/validate/atomic_question_check.md b/skills/kumo-data-science/validate/atomic_question_check.md new file mode 100644 index 000000000..bad9ea6e7 --- /dev/null +++ b/skills/kumo-data-science/validate/atomic_question_check.md @@ -0,0 +1,84 @@ +# Atomic Question Check + +**Goal:** Confirm that the prediction target is a single, well-defined question before investing in graph construction and model training. + +A composite or ambiguous target produces a model that optimizes for nothing clearly, or forces the user to re-do work after training. + +--- + +## The rule + +A request **passes** the atomic question check when: +1. There is exactly **one prediction target** (one column, one label, one outcome). +2. The target is **operationally defined** — the user's words name a concrete outcome ("at least one fraud event", "a cancelled booking"). Which column or value records it is inferred by preflight and confirmed at the gate, not asked (`accept/intake_contract.md`, *named intent, inferred mapping*). +3. The time horizon is **specified** when the prediction is time-dependent. `time_window` is + tri-state: `specified` → pass; `not_critical` (a static task — static link prediction, + static regression/classification, no "in the next N days" semantics) → pass without asking; + `missing` (temporal question, no window) → ASK BACK for the window only. + Static vs. temporal follows `accept/intake_contract.md` (*static or temporal*): a date + column in the data never makes a task temporal. + +A request **fails** when any of the following are true: +- Multiple targets are bundled ("predict X and Y") +- The target is vague or subjective ("risky customers", "high-value users") without a concrete operationalization +- The time horizon is ambiguous for a temporal prediction ("will churn soon") +- The target requires data that is not in the source scope (checked by preflight, not by asking) + +--- + +## Pass examples + +| Request | Why it passes | +|---|---| +| "Predict which customers will churn — no purchase in the next 60 days — using `churned` boolean." | Single target, time horizon defined, operationally concrete. | +| "Predict the `expected_order_amount` for each customer in the next 30 days." | Single regression target, horizon clear. | +| "Predict whether a transaction (`transaction_id`) is fraudulent (`is_fraud` = 1)." | Single binary label, no ambiguity. | +| "Recommend the top 5 products (from `PRODUCTS.product_id`) most likely to be purchased by each customer in the next 7 days." | Single ranking target, K and horizon defined. | + +--- + +## Fail examples + +### Fail 1 — Multiple targets bundled +> "Predict which customers will churn AND which products to recommend to retain them." + +**Problem:** Two distinct tasks (binary classification + recommendation). They require different graph structures and different PQL queries. Split into two separate requests. + +**Suggested fix:** "Please submit these as two separate requests — one for churn prediction, one for retention recommendations." + +--- + +### Fail 2 — Vague/undefined target +> "Identify our high-value customers." + +**Problem:** "High-value" has no operational definition. Is it total spend? Predicted LTV? Number of orders? Without a concrete column or formula, no model can be trained. + +**Suggested fix:** "Please define what 'high-value' means in terms of a concrete column or computable metric. For example: 'Predict which customers will have `total_spend > $500` in the next 90 days.'" + +--- + +### Fail 3 — Ambiguous time horizon +> "Predict which customers are about to churn." + +**Problem:** "About to" is undefined. 7 days? 30 days? 6 months? The time horizon determines the label construction, training cutoff, and feature window. + +**Suggested fix:** "Please specify the prediction horizon. For example: 'Predict which customers will churn in the next 30 days (no purchase within 30 days from today).'" + +--- + +### Fail 4 — Target column not in schema +> "Predict customer satisfaction score." (Preflight finds no satisfaction column or survey table in the source scope.) + +**Problem:** The target cannot be computed from the tables in the source scope. Either the data lives elsewhere, or this is the wrong dataset for the task. + +**Suggested fix:** "The tables in `` do not contain a satisfaction score or any data that could derive one. Please point me at the schema that holds it, or clarify how the target would be computed." + +--- + +## How to handle a failure + +Return a response that: +1. Names the specific failure mode (bundled targets / vague target / ambiguous horizon / missing column). +2. Quotes or paraphrases the problematic part of the request. +3. Gives a concrete example of a fixed version. +4. Does **not** proceed to preflight, the graph gate, or code generation. diff --git a/skills/kumo-data-science/validate/examples/bad_request.md b/skills/kumo-data-science/validate/examples/bad_request.md new file mode 100644 index 000000000..cfae6e7f5 --- /dev/null +++ b/skills/kumo-data-science/validate/examples/bad_request.md @@ -0,0 +1,68 @@ +# Examples: Requests that ASK BACK or REJECT + +## 1. Missing required fields → ASK BACK + +> "Use my sales data to figure out which customers are risky." + +### Node 1a — Required fields: ASK BACK +- `source_scope`: "sales data" does not resolve to a database and schema ✗ +- `prediction_entity`: customers ✓ +- `prediction_target`: "risky" is not an observable outcome ✗ +- `time_window`: `missing` — reads as temporal, no window ✗ + +**Not missing, not to be asked for:** schema, column types, join keys, output table. + +Response to user: + +> "To proceed I need three things: +> +> 1. **Where the data lives** — for example a database and schema, or a folder of files. +> 2. **What 'risky' means** — an observable outcome, e.g. 'a chargeback is filed' or 'no purchase'. +> 3. **The prediction window** — e.g. 'in the next 30 days'." + +Why the reply stops here: the atomic check and preflight need a target and a source. Asking for +schema details now would be asking the user to do the agent's job. + +--- + +## 2. Only the window is missing → ASK BACK for one field + +> "On `USER_DATA.E_COMMERCE`, build a notebook that flags customers who will have a fraud event." + +- `source_scope` ✓ `prediction_entity` ✓ `prediction_target` ✓ `time_window`: `missing` ✗ + +> "One thing: over what window should 'will have a fraud event' be judged — e.g. the next 30 days?" + +Nothing else is asked. Not the output table, not the columns. + +--- + +## 3. Source does not resolve → ASK BACK + +> "Train on `USER_DATA.AD_CLICKS`, recommend paths per cookie." + +Preflight: schema not found in `USER_DATA`. + +> "I can't find a schema named `AD_CLICKS` in `USER_DATA`. Could you check the database and schema name? I won't guess a similar one." + +--- + +## 5. Full data-science request → REJECT + +> "Here is our Snowflake semantic view. Figure out what we should predict, choose the best target +> and horizon, build the best graph, tune the model, and deploy it." + +> "I can't choose the predictive problem or optimise the model on your behalf. To generate training +> code I need: where the data lives, the entity that receives a prediction, the observable outcome +> (or interaction), and the time window. I can infer tables, keys and links from there and show +> them to you for confirmation. Which single question would you like to start with?" + +No artifact is generated. The agent does not propose a target, horizon, or graph. + +--- + +## 6. Bundled goals → REJECT + +> "Predict which customers will churn AND which products to recommend to retain them." + +Two tasks, two graphs, two PQL statements — one session covers one predictive task. Ask the user to submit them separately. diff --git a/skills/kumo-data-science/validate/examples/good_request.md b/skills/kumo-data-science/validate/examples/good_request.md new file mode 100644 index 000000000..86d53db4a --- /dev/null +++ b/skills/kumo-data-science/validate/examples/good_request.md @@ -0,0 +1,106 @@ +# Example: Good Request (passes intake, then two confirmations) + +## Input + +> "Create a notebook that trains a Kumo model for our fraud use case. The data is in +> `USER_DATA.E_COMMERCE`. Entity: each customer. Outcome: at least one fraud event. Window: next 30 days." + +Note what is **not** in the request: no column names, no types, no join keys, no output table. +That is a complete training request. + +--- + +## Node 1a — Required fields: PASS +- operation: `train` ✓ +- `source_scope`: `USER_DATA.E_COMMERCE` ✓ +- `prediction_entity`: each customer ✓ +- `prediction_target`: at least one fraud event (positive-label condition stated) ✓ +- `time_window`: `specified`, 30 days ✓ + +## Node 1b — Atomic question: PASS +- Single target, operationally defined, horizon explicit ✓ + +## Node 1c — Preflight & graph gate + +Agent runs read-only preflight (`accept/preflight.md`) and replies — then stops: + +> Source scope: `USER_DATA.E_COMMERCE` [supplied] — 7 tables found +> Table mode: all-connected [default] — 7 of 7 tables reachable from the seeds +> +> **Metadata** +> - customers: primary key CUSTOMER_ID [inferred] +> - fraud_events: primary key EVENT_ID [inferred] +> - fraud_events: time column EVENT_AT [inferred] +> - bookings: primary key BOOKING_ID [inferred]; time column BOOKED_AT [inferred] +> +> **Links** +> - fraud_events.CUSTOMER_ID -> customers.CUSTOMER_ID [inferred] +> - bookings.CUSTOMER_ID -> customers.CUSTOMER_ID [inferred] +> +> **Excluded columns** +> - bookings.CANCELLED_AT [inferred] — not knowable at the row's BOOKED_AT +> - bookings.NIGHTS_STAYED [inferred] — realized duration of a stay that has not ended +> +> **Not found** +> - payment_lines: no primary key inferred [inferred] — treated as keyless; edge source only +> +> Confirm, or correct any line — your values take precedence. + +User: "Confirmed." + +## Node 1d — Task interpretation gate + +Agent replies — then stops: + +> - Entity: one prediction per customers.CUSTOMER_ID +> - Outcome: at least one fraud_events.EVENT_ID occurs +> - Window: 0 to 30 days after the prediction anchor +> - Task type: temporal binary classification +> - Output: notebook only, no predictions written +> - Embeddings: not requested +> +> Shall I generate the notebook? + +User: "Yes." + +## ACCEPT → Step 2 + +Structured handoff payload to `kumoai-reference/SKILL.md`: +``` +operation: train +source_scope: USER_DATA.E_COMMERCE +table_mode: all-connected +tables: + - name: CUSTOMERS primary_key: CUSTOMER_ID + - name: FRAUD_EVENTS primary_key: EVENT_ID time_column: EVENT_AT + - name: BOOKINGS primary_key: BOOKING_ID time_column: BOOKED_AT + excluded_columns: [CANCELLED_AT, NIGHTS_STAYED] # carried into column_names +excluded_columns_confirmed: true +links: + - FRAUD_EVENTS.CUSTOMER_ID -> CUSTOMERS.CUSTOMER_ID + - BOOKINGS.CUSTOMER_ID -> CUSTOMERS.CUSTOMER_ID +target: + entity: CUSTOMERS.CUSTOMER_ID + definition: at least one FRAUD_EVENTS row in window + horizon_days: 30 + task_type: temporal binary classification +metric: AUC-ROC (default) +artifact: notebook — fit, then predict locally (no destination); one blocking fit call +``` + +If the user had written "…and write predictions to `SCORES_DB.PUBLIC.FRAUD_SCORES`", the Output +line at Node 1d would read "predictions written to `SCORES_DB.PUBLIC.FRAUD_SCORES` in the same +notebook", the payload would carry `output_location: SCORES_DB.PUBLIC.FRAUD_SCORES`, and the +notebook's predict call would target it. It is never asked for. + +--- + +## Variant: the same request in two turns + +Turn 1: "We want a 30-day fraud model on `USER_DATA.E_COMMERCE` — flag customers who will have any +fraud event." → all four required fields present; agent goes straight to Node 1c. + +Turn 1: "Build me a fraud model for customers." → `source_scope` and `time_window` missing; +agent asks for exactly those two, nothing about schema or output. + +--- From 65a7ea298bee9d258c9b49b5b13063531a292401 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Mon, 9 Nov 2026 13:33:59 -0600 Subject: [PATCH 04/35] Refresh the published copy for #250's kumo-model-explanations change Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- skills/kumo-data-science/SKILL.md | 2 +- .../kumo-model-explanations/SKILL.md | 11 +- .../references/artifacts.md | 311 ++++++++---------- 3 files changed, 142 insertions(+), 182 deletions(-) diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index aab8c4229..242ca2116 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-06-113fde01`** +**`SKILL-BUILD: 2026-11-09-c2384fd6`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/kumo-model-explanations/SKILL.md b/skills/kumo-data-science/kumo-model-explanations/SKILL.md index 35b9238fe..f62f4f10f 100644 --- a/skills/kumo-data-science/kumo-model-explanations/SKILL.md +++ b/skills/kumo-data-science/kumo-model-explanations/SKILL.md @@ -13,9 +13,9 @@ with its in-process `kumo.engine` runtime and retained local run files. ## Establish the run and question - Reuse the user's task, model, graph, and run identity. Do not restart training intake. -- For a fresh `training_result`, `Path(training_result.bundle_dir).parent` identifies that fit's - run directory in the reviewed implementation. Otherwise use the retained run directory the - user supplied. A downloaded model bundle is not the original training run. +- For a fresh `training_result`, use `training_result.explanations`. Otherwise reopen the + explicit retained run with `kumo.TrainingExplanations.load(run_dir)`. A downloaded model + bundle is not the original training run. - If several runs or trainer/refit directories exist, list candidates and resolve which one the user means. Never silently select the newest directory or combine different runs. - Identify the task type, target/positive class, explanation stage, and fit/refit used. @@ -45,8 +45,9 @@ from the existence of `disable_explain`. column-analysis workflow; forecasting is not in the reviewed producer's column-analysis list. - Gradient scoring supports more task types, but the shared trainer skips TWO_TOWER link models. The runtime's explainer task list can narrow support further. -- There is no public `kumo.Trainer.explain()` or `training_result.explanations` accessor in - this API. Do not call private trainers or enable prediction-time XAI through invented options. +- `training_result.explanations` retrieves retained TEST-stage artifacts and provides gradient + and cohort plots. There is no public `kumo.Trainer.explain()` or standalone regeneration API. + Do not call private trainers or enable prediction-time XAI through invented options. - An inspection request does not authorize another fit. If required files are absent, diagnose the reason first. Generate an additional fit only when the user has requested that run. diff --git a/skills/kumo-data-science/kumo-model-explanations/references/artifacts.md b/skills/kumo-data-science/kumo-model-explanations/references/artifacts.md index 4cffba476..8373bfd94 100644 --- a/skills/kumo-data-science/kumo-model-explanations/references/artifacts.md +++ b/skills/kumo-data-science/kumo-model-explanations/references/artifacts.md @@ -1,169 +1,137 @@ # Explanation artifact workflow -## Find the correct files +## Open the selected run -In the existing Snowflake notebook, use the fresh fit's result, or replace `run_dir` with an -explicit retained run directory. Do not change `kumo.configure` while inspecting an existing run. +In the existing notebook, reuse the result of the requested fit: ```python -from pathlib import Path -import json -import pandas as pd import kumo.engine as kumo print("Kumo runtime:", kumo.__version__) +explanations = training_result.explanations +print("Run:", explanations.run_root) +print("Selected TEST trial:", explanations.trial_root) +print(explanations.availability) +``` + +To reopen an explicitly retained run instead, replace the assignment with +`explanations = kumo.TrainingExplanations.load("/path/to/retained/run")`. +Do not change `kumo.configure`, search other runs, or train while inspecting existing files. +If these APIs are absent from the executing build, report the runtime version and unavailable +interface rather than calling private trainers. + +The loader resolves one `models//automl/best_trainer.json` within the supplied run and +reads the selected trial's `config.json` refit state. TEST explanations use the recorded +train+validation refit (`_refit_trainval`), otherwise the original trial; they never select the +full-data refit. Directory presence alone does not select a refit. Copied runs resolve within +the supplied root instead of following old absolute paths. Invalid or ambiguous selection +raises `kumo.InvalidRunError`; resolve the intended run rather than choosing the newest match. +Retain this selection metadata as well as the artifacts to reopen a run through the API. + +Availability keys are `gradient`, `cohorts`, and `samples`. Each entry reports `available`, +`path`, `reason`, and `generation_failure`. Availability checks presence, not contents; +a generation failure can leave readable partial artifacts. Missing files are not zero-valued +explanations. Reopening cannot recover generation status captured only during fit, so missing +artifacts may have an unknown cause. Readers raise `kumo.ExplanationUnavailableError` for +missing artifacts and `kumo.InvalidExplanationError` for malformed data. + +Published model bundles exclude these files. `Trainer.load(bundle)` leaves explanations +unavailable with reason `model_only`; retain `explanations.run_root` separately. -run_dir = Path(training_result.bundle_dir).parent.resolve() # fresh fit only -if not run_dir.is_dir(): - raise FileNotFoundError(f"Training run is unavailable: {run_dir}") - -gradient_files = sorted(run_dir.rglob("explain_test/scores.csv")) -column_files = sorted(run_dir.rglob("column_analysis/test/column_analysis.json")) -for kind, paths in (("gradient", gradient_files), ("column analysis", column_files)): - print(kind, len(paths)) - for path in paths: - print(path.relative_to(run_dir)) -for path in sorted(run_dir.rglob("best_trainer.json")): - print("selection metadata:", path.relative_to(run_dir)) - print(json.loads(path.read_text())) +## Read and plot gradient scores + +```python +from IPython.display import display + +if explanations.availability["gradient"].available: + scores = explanations.gradient_scores().to_pandas() # table, column, score; descending + if scores.empty: + print("No gradient rows were recorded.") + else: + print("Recorded score sum:", scores["score"].sum()) + display(scores.head(20)) + figure, axis = explanations.plot_gradient(top_k=20) + axis.set_title(f"TEST gradient scores — {explanations.trial_root}") + display(figure) +else: + print(explanations.availability["gradient"]) ``` -An empty list is “no matching artifact found in this run”, not “no important features”. -Use `best_trainer.json`, the run configuration, and logs to associate files with the selected -trial. Test explanations use the train+validation refit when available, otherwise the original -fit; do not label them as full-data-refit explanations. Keep trainer and refit paths visible. -Selection metadata may contain paths from the original host after a copy. Do not follow those -paths into another run; resolve against the retained run or report the missing mapping. +The reader uses `explain_test/scores.csv` from the selected trial and returns the shared +`GraphGradientScore` schema. It validates the CSV rather than treating extra columns or +malformed rows as features. The producer aggregates gradients within examples using +gradient-based weights, averages across examples, then normalizes across all table-column +scores when the total is positive. Scores are written with six decimal places; a positive +recorded total is approximately one, while all-zero output remains zero. -Choose an exact file from the inventory, preserving any refit subdirectory. The examples below -use `gradient_path` and `column_path` assigned to those selected `Path` objects; they deliberately -do not default to the first match. +`plot_gradient()` rereads that file, sorts before selecting the top N, labels bars +`table.column`, and preserves recorded values. Do not type values copied from another output, +renormalize a top-20 subset, or present scores as percentages. Report the run and TEST stage. +The helpers return figures and axes without calling `show()`. In notebooks use `display(figure)`; +in scripts use `figure.savefig("training-importance.png", bbox_inches="tight")` when saving is +requested, or `matplotlib.pyplot.show()` for a window. Plotting requires Matplotlib +(`kumo-engine[plots]`). -## Read gradient scores +## Read and plot column cohorts -The reviewed CSV schema is `table`, `column`, `score`. Extra columns may exist, for example a -serialized dataframe index; do not use those as features. +The reader validates `column_analysis/test/column_analysis.json` and returns a shared +`ColumnAnalysisOutput`. Discover the recorded task, complete `(table_name, hop, col_name)` keys, +and class labels before selecting a chart: ```python -def read_gradient_scores(path): - frame = pd.read_csv(path) - required = ["table", "column", "score"] - missing = set(required) - set(frame.columns) - if missing: - raise ValueError(f"Unexpected gradient schema in {path}: missing {sorted(missing)}") - frame = frame[required].copy() - frame["score"] = pd.to_numeric(frame["score"], errors="raise") - if frame["score"].isna().any() or not frame["score"].map( - lambda value: float("-inf") < value < float("inf") - ).all(): - raise ValueError(f"Non-finite gradient scores in {path}") - if (frame["score"] < 0).any(): - raise ValueError(f"Unexpected negative scores in {path}; check producer semantics") - return frame.sort_values("score", ascending=False).reset_index(drop=True) - -scores = read_gradient_scores(gradient_path) -if scores.empty: - print("No gradient rows were recorded.") +if explanations.availability["cohorts"].available: + analysis = explanations.column_analysis() + print("Recorded task:", analysis.task_type) + print("Cohort keys:", explanations.cohort_keys()) else: - print("Recorded score sum:", scores["score"].sum()) - display(scores.head(20)) + print(explanations.availability["cohorts"]) ``` -The producer aggregates gradients within examples using gradient-based weights, averages across -examples, then normalizes across all table-column scores when the total is positive. A positive -normalized total is approximately one; CSV rounding can change that sum, and an all-zero output -remains zero. Do not renormalize a -top-20 subset and present it as a full-graph percentage. Bar charts should label values “gradient -score”, include table and column, and identify the run/stage. +After choosing a key from that output, inspect its labels: -### Plot from the file, not from memory +```python +key = ("users", 0, "age") # replace with the table/hop/column you chose +print("Classes:", explanations.classes(key)) +``` -Plot code reads the selected run's file when it runs. Do not type values copied from an earlier -output, a printed table or a retained summary: those may belong to another run. `scores.csv` is -not necessarily sorted, so sort descending before taking the top N. Label each bar `table.column`; -the same column name (for example `created_at`) can appear in two tables, and a bare column name -merges them. Plot the recorded scores unchanged, with no rescaling, then state the run and stage. -A cohort chart keeps the producer's cohort order and keeps `N/A` and `OTHER(*)` as bars of their -own. +For binary positive-class or regression vectors, `classes(key)` is empty: -## Read column cohorts +```python +cohort = explanations.cohort(key) +figure, (prediction_axis, population_axis) = explanations.plot_cohort(key) +display(figure) +``` -The JSON contains `task_type` and a `cohorts` list. Each entry identifies `table_name`, `hop`, -`col_name`, and `stype`; its inner `cohorts` list contains bucket labels. `populations` is a list -of relative shares, not integer row counts. `predictions` and optional `targets` are either -lists or mappings from class labels to lists. The latter must be interpreted one class at a time. -Nonempty cohort labels require prediction data; an empty prediction mapping is incomplete, -not an empty cohort result. Strings are not valid series, even when their lengths match. +For class-keyed multiclass or multilabel outputs, use this alternative with an explicitly +chosen label from `classes(key)`: ```python -def read_column_cohorts(path): - payload = json.loads(Path(path).read_text()) - if not isinstance(payload, dict) or not isinstance(payload.get("cohorts"), list): - raise ValueError(f"Unexpected column-analysis schema in {path}") - rows = [] - for entry in payload["cohorts"]: - if not isinstance(entry, dict): - raise ValueError(f"Unexpected cohort entry in {path}") - labels = entry.get("cohorts") - shares = entry.get("populations") - if not isinstance(labels, list) or not isinstance(shares, list): - raise ValueError(f"Cohort labels and populations must be lists in {path}") - predictions = entry.get("predictions") - targets = entry.get("targets") - if not isinstance(predictions, (list, dict)): - raise ValueError(f"Predictions must be a list or class mapping in {path}") - if targets is not None and not isinstance(targets, (list, dict)): - raise ValueError(f"Targets must be a list or class mapping in {path}") - predictions = predictions if isinstance(predictions, dict) else {None: predictions} - targets = targets if isinstance(targets, dict) else {None: targets} - if labels and not predictions: - raise ValueError(f"Missing predictions for nonempty cohorts in {path}") - if len(shares) != len(labels): - raise ValueError(f"Cohort/population length mismatch in {path}") - for name, series in (("predictions", predictions), ("targets", targets)): - for values in series.values(): - if name == "targets" and values is None: - continue - if not isinstance(values, list): - raise ValueError(f"Each {name} series must be a list in {path}") - if len(values) != len(labels): - raise ValueError(f"Cohort/{name} length mismatch in {path}") - for class_label, values in predictions.items(): - observed = targets.get(class_label) - for index, label in enumerate(labels): - rows.append({ - "table": entry["table_name"], "hop": entry["hop"], - "column": entry["col_name"], "class": class_label, - "cohort": label, "population_share": shares[index], - "mean_prediction": values[index], - "mean_target": None if observed is None else observed[index], - "prediction_score": entry["prediction_score"], - "target_score": entry.get("target_score"), - }) - return payload.get("task_type"), pd.DataFrame(rows) - -task_type, cohorts = read_column_cohorts(column_path) -print("Recorded task:", task_type) -if cohorts.empty: - print("No column cohorts were recorded.") -else: - display(cohorts.head(30)) +class_name = "True" # replace with your chosen recorded class label +cohort = explanations.cohort(key, class_name=class_name) +figure, (prediction_axis, population_axis) = explanations.plot_cohort( + key, class_name=class_name) +display(figure) ``` -Select one table/column/hop/class before plotting means. Preserve cohort ordering from the -producer. `COUNT(*)` is a relationship-count cohort; `OTHER(*)` groups remaining categories; -`N/A` describes missing values. None is a literal user feature to remove automatically. -Do not derive exact counts from population shares without the correct denominator. Scores are -weighted standard deviations before normalization, not measures of predictive accuracy. +`cohort()` returns `ColumnCohorts`; class selection validates the label while retaining the +full class dictionaries. The upper plot compares mean predictions and observed targets when +recorded; the lower plot shows population shares separately. These are relative shares, not +integer row counts. Do not derive counts without the correct denominator or fabricate a +missing negative-class vector. Preserve producer order, including `N/A` (missing values), +`OTHER(*)` (remaining categories), and `COUNT(*)` (relationship counts); none is a literal +user feature to remove automatically. Scores measure weighted between-cohort standard +deviation before separate normalization, not predictive accuracy or gradient attribution. ## Sampled subgraphs The same post-training step writes per-entity subgraphs to `subgraph_collector/test/` in the selected trainer directory: `strategy_mappings.json` plus `subgraph_json/.json`, one file -per sampled test entity. Take the trainer name from `best_trainer.json` (the last component of its -`best_trainer` path) and use its `_refit_trainval` sibling when that directory exists. Do not glob -every `version_*` directory or every run: only the selected trainer's files describe this model. The -folder also holds other collector formats (`nested_json/`, `output/`, `rfm/`) and +per sampled test entity. The API resolves the selected trainer from `best_trainer.json` and +its recorded `_refit_trainval` state as described above. TEST explanations use the +train+validation refit when recorded, otherwise the original fit. If the published bundle uses a +full-data refit, these artifacts describe the earlier fit, not the published model's parameters. +Do not glob every `version_*` directory or every run; inspect only the selected TEST trial. +The folder also holds other collector formats (`nested_json/`, `output/`, `rfm/`) and `global_aggr_feature/`; read `subgraph_json/` only. The files cover a bounded sample, not the test set. For each selection strategy (for binary @@ -185,53 +153,40 @@ Each file is a `kumoapi.explain.Subgraph`: - `score` is a gradient magnitude divided by the largest one in that subgraph, so it is comparable within one file, not across files or with `scores.csv`. It may be null. -JSON object keys are strings: index `nodes`, `edges` and `features` with `str(node_id)`; ids inside -`edges` and `tables` lists are integers. +List sampled graph references without loading their bodies: + +```python +samples = explanations.samples() +if not samples: + print("No sampled graphs found in:", explanations.trial_root) +for reference in samples: + print(reference.artifact_id, reference.categories, reference.original_ids, + reference.available) +``` + +`samples()` lists references without loading graph bodies. Collector-local `artifact_id` values +are not entity primary keys; `categories` and optional `original_ids` preserve the sampling +metadata. A reference may name a missing graph, so check `available`. A graph can also be read +without mapping metadata; arbitrary original-key lookup is not provided. + +Choose an artifact ID from this inventory; do not silently select the first sample: ```python -from pathlib import PurePosixPath - -def selected_trainer_dir(run_dir): - metadata = sorted(Path(run_dir).rglob("best_trainer.json")) - if len(metadata) != 1: - raise ValueError(f"Expected one best_trainer.json under {run_dir}: {metadata}") - name = PurePosixPath(json.loads(metadata[0].read_text())["best_trainer"]).name - for wanted in (f"{name}_refit_trainval", name): - found = [p for p in metadata[0].parent.rglob(wanted) if p.is_dir()] - if len(found) == 1: - return found[0] - raise FileNotFoundError(f"Trainer {name} is not under {metadata[0].parent}") - -def subgraph_tables(subgraph): - store = subgraph["subgraph_store"] - table_of = {node_id: table for table, ids in store["tables"].items() for node_id in ids} - nodes = pd.DataFrame([dict(node, table=table_of.get(int(key))) - for key, node in store["nodes"].items()]) - edges = pd.DataFrame([{"src": int(src), "relation": relation, "dst": dst} - for src, relations in store["edges"].items() - for relation, dsts in relations.items() for dst in dsts]) - by_node = (subgraph.get("feature_store") or {}).get("features") or {} - features = pd.DataFrame([dict(feature, node_id=int(key)) - for key, columns in by_node.items() for feature in columns.values()], - columns=["node_id", "table_name", "name", "value", "stype", "score"]) - return store["nodes"][str(store["root_id"])], nodes, edges, features - -subgraph_dir = selected_trainer_dir(run_dir) / "subgraph_collector" / "test" -paths = sorted((subgraph_dir / "subgraph_json").glob("*.json"), key=lambda p: int(p.stem)) -if not paths: - print(f"No sampled subgraphs in {subgraph_dir}. Check disable_explain in this run's plan " - "and the fit log for a post-training explanation error.") +artifact_id = "0" # replace with your chosen available collector ID +reference = next(item for item in samples if item.artifact_id == artifact_id) +if reference.available: + graph = explanations.sample(reference.artifact_id) # typed kumoapi Subgraph + print(graph.subgraph_store, graph.feature_store) else: - mappings = json.loads((subgraph_dir / "strategy_mappings.json").read_text()) - subgraphs = {int(p.stem): json.loads(p.read_text()) for p in paths} - print(len(subgraphs), "subgraphs; strategies:", sorted(mappings["mappings"])) - root, nodes, edges, features = subgraph_tables(subgraphs[min(subgraphs)]) - print("root entity:", root["pkey_name"], "=", root["pkey"]) - display(nodes, edges, features.sort_values("score", ascending=False, na_position="last")) + print("Sample unavailable:", reference.path) ``` -Absent or empty `subgraph_json/` is not "no neighbourhood": report the path and check the items -in *If artifacts are missing*. The collector skips forecasting, multilabel ranking, and +The typed stores use integer node keys, unlike raw JSON's string object keys. Either store can +be absent. The API preserves structure, feature values, scores, timestamps, and aggregated +nodes; it does not render an interactive sampled graph. + +Absent or empty `subgraph_json/` is not "no neighbourhood": report the path, check +`disable_explain` and post-training errors, and follow *If artifacts are missing*. The collector skips forecasting, multilabel ranking, and link-prediction models other than the hybrid, ID and shallow-RHS modules. These files carry entity primary keys and raw feature values of real rows. Review them before sharing, and keep them out of shared locations unless the user asks. @@ -261,6 +216,10 @@ Do not invoke internal `best_trainer.explain` as customer API code. Source: Kumo modules, as shipped in the runtime: +- `kumo.engine.explain.results.TrainingExplanations`: lazy availability, typed readers, + sample references and plotting; `kumo.engine.explain._artifacts`: retained-run selection + and artifact validation; `kumo.engine.explain.plotting`: plotting semantics. + - `kumo.engine.trainer.trainer.Trainer.fit`: post-training dispatch, disable flag, error handling, local result fields and run directory. - `kumo_core.train.trainer.Trainer.set_explainers`, `kumo_core.train.trainer.Trainer.explain`, From f3fb77c949036141e7c41eed180ecaba64df6fee Mon Sep 17 00:00:00 2001 From: igutierrez Date: Mon, 9 Nov 2026 13:34:31 -0600 Subject: [PATCH 05/35] publish_nvskills: refuse catalog collisions and signed names at any depth A catalog file whose target the package already ships (e.g. a second SKILL.md) now fails the build instead of silently replacing it, and BENCHMARK.md or skill.oms.sig is refused anywhere in the catalog files. The build also fails when the package itself carries either name, which publish() would otherwise copy over the CI-written file while --check, which skips those names, stayed green. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- agent-skills/scripts/publish_nvskills.py | 13 +++++++-- agent-skills/tests/test_publish_nvskills.py | 31 +++++++++++++++++++-- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/agent-skills/scripts/publish_nvskills.py b/agent-skills/scripts/publish_nvskills.py index 2d68f68b1..58b5b6680 100644 --- a/agent-skills/scripts/publish_nvskills.py +++ b/agent-skills/scripts/publish_nvskills.py @@ -22,9 +22,10 @@ * the package, rebuilt from `agent-skills/skills/` on every run; * the catalog files that NVSkills requires and that we write: `skill-card.md` and `evals/*.json`. Their source is `agent-skills/nvskills/kumo-data-science/`, and they are - copied in on every run; + copied in on every run. A catalog file may not replace a package file; * `BENCHMARK.md` and `skill.oms.sig`, which NVSkills CI writes into the pull request. They are - never edited here. A refresh leaves them in place, so they describe the previous content until + never edited here, and the build fails if the package or the catalog files carry either + name at any depth. A refresh leaves them in place, so they describe the previous content until NVSkills CI runs again. The catalog then keeps serving the last signed version rather than dropping the skill. @@ -61,13 +62,19 @@ def build(out: Path, catalog_files: Path = CATALOG_FILES) -> Path: errors = package_skill.unresolved(pkg) if errors: raise SystemExit(f'{HOST}: {len(errors)} unresolved reference(s): {errors[:3]}') + # A signed name in the package would overwrite what NVSkills CI wrote, unseen by --check. + signed = [p.relative_to(pkg).as_posix() for p in pkg.rglob('*') if p.name in SIGNED] + if signed: + raise SystemExit(f'{HOST} package ships {signed}: written by NVSkills CI, not by hand') if catalog_files.is_dir(): for path in sorted(catalog_files.rglob('*')): if path.is_file(): rel = path.relative_to(catalog_files) - if rel.as_posix() in SIGNED: + if path.name in SIGNED: raise SystemExit(f'{path}: {rel} is written by NVSkills CI, not by hand') target = pkg / rel + if target.exists(): + raise SystemExit(f'{path}: {rel} is already in the {HOST} package') target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(path, target) return pkg diff --git a/agent-skills/tests/test_publish_nvskills.py b/agent-skills/tests/test_publish_nvskills.py index 67f3bae0d..f7831de2c 100644 --- a/agent-skills/tests/test_publish_nvskills.py +++ b/agent-skills/tests/test_publish_nvskills.py @@ -64,12 +64,39 @@ def test_check_ignores_the_build_stamp_but_not_content(tmp_path: Path, catalog: assert publish_nvskills.stale(dest, catalog) == ['~ SKILL.md'] -def test_catalog_files_cannot_supply_signed_files(tmp_path: Path, catalog: Path) -> None: - (catalog / 'skill.oms.sig').write_text('forged\n') +@pytest.mark.parametrize('rel', ['skill.oms.sig', 'evals/skill.oms.sig', 'evals/BENCHMARK.md']) +def test_catalog_files_cannot_supply_signed_files(tmp_path: Path, catalog: Path, + rel: str) -> None: + (catalog / rel).write_text('forged\n') with pytest.raises(SystemExit, match='written by NVSkills CI'): publish_nvskills.publish(tmp_path / 'kumo-data-science', catalog) +@pytest.mark.parametrize('rel', ['SKILL.md', 'hosts/generic/adapter.md']) +def test_catalog_files_cannot_replace_package_files(tmp_path: Path, catalog: Path, + rel: str) -> None: + (catalog / rel).parent.mkdir(parents=True, exist_ok=True) + (catalog / rel).write_text('replaced\n') + with pytest.raises(SystemExit, match='already in the generic package'): + publish_nvskills.publish(tmp_path / 'kumo-data-science', catalog) + + +def test_the_package_cannot_ship_signed_files(tmp_path: Path, catalog: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + build = publish_nvskills.package_skill.build + + def with_signature(host: str, dist: Path) -> Path: + pkg = build(host, dist) + (pkg / 'kumoai-reference' / 'BENCHMARK.md').write_text('forged\n') + return pkg + + monkeypatch.setattr(publish_nvskills.package_skill, 'build', with_signature) + dest = tmp_path / 'kumo-data-science' + with pytest.raises(SystemExit, match='written by NVSkills CI'): + publish_nvskills.publish(dest, catalog) + assert not dest.exists() + + def test_the_published_copy_matches_the_source() -> None: """The copy NVIDIA/skills syncs from main is rebuilt in the same PR as the change. From 13c6910eefcaf9e12d966cb452f4b3f3f65c7d00 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Mon, 9 Nov 2026 13:34:54 -0600 Subject: [PATCH 06/35] agent-skills README: list what the first publish needs; name /ok to test and sig drift The section described the catalog files, the onboarding entry and the dispatch secret as if they existed. It now lists them as prerequisites, says the nv-skills-ci[bot] signature commit needs /ok to test before kumo-core-gpu-ci runs, and that an unsigned merge, a revert included, leaves the catalog serving the last signed copy. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- agent-skills/README.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/agent-skills/README.md b/agent-skills/README.md index bea871dcc..50ca03d70 100644 --- a/agent-skills/README.md +++ b/agent-skills/README.md @@ -146,10 +146,24 @@ Whatever is on `main` is what the catalog publishes. To change the skill: `tests/test_publish_nvskills.py` fails the PR if you forget. It ignores the `SKILL-BUILD` stamp, so a rebuild on an unrelated commit is not required. 3. A maintainer comments `/nvskills-ci` on the PR. NVSkills CI runs the evals and pushes - `BENCHMARK.md` and `skill.oms.sig`. A later push to the PR needs another `/nvskills-ci`. -4. Merge. The catalog picks the change up within the hour. A PR merged without step 3 leaves the - catalog on the last signed version. - -The catalog side is `components.d/kumo-engine.yml` in NVIDIA/skills, which points at -`skills/kumo-data-science`. Its `verify-source-onboarding` check fails until a signed copy with -`skill-card.md`, `BENCHMARK.md` and an eval dataset is on `main` here. + `BENCHMARK.md` and `skill.oms.sig` as a commit by `nv-skills-ci[bot]`. copy-pr-bot does not + trust that commit, so `kumo-core-gpu-ci` runs only after a maintainer comments + `/ok to test `. A later push to the PR needs another `/nvskills-ci`. +4. Merge. The catalog picks the change up within the hour. + +A change merged without step 3, a revert included, is "sig drift" to the catalog sync: it keeps +the last signed copy, so the catalog serves content that `main` no longer has until the next signed +merge. Run `/nvskills-ci` on reverts too. + +### Before the first publish + +None of this exists yet; the catalog publishes nothing until all of it does: + +- [ ] `agent-skills/nvskills/kumo-data-science/skill-card.md` +- [ ] `agent-skills/nvskills/kumo-data-science/evals/evals.json` +- [ ] the `NVSKILLS_CI_DISPATCH_TOKEN` repository secret, from the NVSkills CI maintainers; + `request-nvskills-ci.yml` fails without it +- [ ] a signed copy on `main`: a PR through steps 1–4 whose `BENCHMARK.md` says PASS +- [ ] the onboarding PR to NVIDIA/skills adding `components.d/kumo-engine.yml`, pointing at + `skills/kumo-data-science`. Its `verify-source-onboarding` check passes only once the + signed copy is on `main`. From 8976fad1c366bec2670a97f4101157d57a74b5d7 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Mon, 9 Nov 2026 13:39:42 -0600 Subject: [PATCH 07/35] generic runtime: keep each location's project, catalog, warehouse and credentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - BigQuery: bq_location takes the project; destination and population use OUTPUT_PROJECT / POP_PROJECT for connector, location, staging and read-back. - Databricks: read-back and staging use OUTPUT_CATALOG / POP_CATALOG. - Snowflake: SQL connections pass SNOWFLAKE_WAREHOUSE; read-back quotes the table name, which Kumo writes case-preserved (kumo_finetuning_sdk.md §9.1). - S3 with an explicit S3Identity: Arrow reads and writes take the same keys. - File population staging streams batches instead of materializing the entity columns. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- agent-skills/skills/hosts/generic/runtime.md | 78 ++++++++++++------- .../hosts/generic/runtime.md | 78 ++++++++++++------- 2 files changed, 100 insertions(+), 56 deletions(-) diff --git a/agent-skills/skills/hosts/generic/runtime.md b/agent-skills/skills/hosts/generic/runtime.md index b31cb4f17..cea0c6b06 100644 --- a/agent-skills/skills/hosts/generic/runtime.md +++ b/agent-skills/skills/hosts/generic/runtime.md @@ -69,7 +69,7 @@ connector = kumo.GCSConnector(root_dir=SOURCE_URI) # 'gs://bucket/pref # credential. No billing_project_id (§1). location= is the dataset's own, read by bq_location below. connector = kumo.BigQueryConnector(project_id=PROJECT_ID, dataset_id=DATASET_ID, identity=kumo.BigQueryIdentity(), - location=bq_location(DATASET_ID)) + location=bq_location(PROJECT_ID, DATASET_ID)) # Databricks — exactly one of warehouse_id / cluster_id connector = kumo.DatabricksConnector( catalog=CATALOG, schema_name=SCHEMA, identity=kumo.DatabricksIdentity(), @@ -90,16 +90,21 @@ else — `EU`, or a single region such as `us-central1`, which is not `US` — f not-found, and the prediction write only after training, at the sink's location check. Whenever any connector in the notebook is BigQuery — source, population or destination — define this once, in the connect step before the first connector, and pass -`location=bq_location()` to every `BigQueryConnector`: +`location=bq_location(, )` to every `BigQueryConnector`: ```python from google.cloud import bigquery -def bq_location(dataset): +def bq_location(project, dataset): """The location a dataset lives in; the dataset must already exist.""" - return bigquery.Client(project=PROJECT_ID).get_dataset(f'{PROJECT_ID}.{dataset}').location + return bigquery.Client(project=project).get_dataset(f'{project}.{dataset}').location ``` +Each BigQuery location keeps the project it was confirmed with: `PROJECT_ID` for the source, +`OUTPUT_PROJECT` and `POP_PROJECT` for the destination and the population, equal to `PROJECT_ID` +only when the confirmed names say so. Use that project for its connector, location, staging query +and read-back. + Name file and object-store tables without their suffix — `connector['accounts']` for `accounts.parquet` — and use those names in the tables, edges and PQL (`adapter.md` §3). @@ -112,6 +117,17 @@ identity = kumo.S3Identity(access_key_id=os.environ[AWS_KEY_ID_VAR], secret_access_key=os.environ[AWS_SECRET_VAR]) ``` +Arrow does not see that identity. Every `pyarrow` read or write of S3 in §3 and §3.1 then takes +the same keys, with the path written without `s3://`: + +```python +import pyarrow.fs +s3 = pyarrow.fs.S3FileSystem(access_key=os.environ[AWS_KEY_ID_VAR], + secret_key=os.environ[AWS_SECRET_VAR]) +written = ds.dataset(OUTPUT_LOCATION.removeprefix('s3://'), format=OUTPUT_FORMAT, + filesystem=s3).count_rows() +``` + The Snowflake factory `from_spcs_env` and Snowflake's notebook secret store belong to Snowflake's own runtime. Emit them only when the user says the code runs there. @@ -144,8 +160,9 @@ output_config = kumo.PredictionOutputConfig( OUTPUT_LOCATION = OUTPUT_PARENT + OUTPUT_LEAF # the confirmed path, for read-back ``` -A BigQuery destination's connector is scoped to `OUTPUT_DATASET` and takes -`location=bq_location(OUTPUT_DATASET)` (§2), which may differ from the source's. +A BigQuery destination's connector is scoped to `OUTPUT_PROJECT` and `OUTPUT_DATASET` and takes +`location=bq_location(OUTPUT_PROJECT, OUTPUT_DATASET)` (§2), which may differ from the source's. +A Databricks destination's connector is scoped to `OUTPUT_CATALOG` and `OUTPUT_SCHEMA`. Rooting the connector at the destination itself and passing a `table_name` writes one level below it, to a path the user never confirmed. A directory destination must be empty or absent; @@ -163,8 +180,8 @@ written = ds.dataset(OUTPUT_LOCATION, format=OUTPUT_FORMAT).count_rows() # full # BigQuery from google.cloud import bigquery -written = next(iter(bigquery.Client(project=PROJECT_ID).query( - f'SELECT COUNT(*) FROM `{PROJECT_ID}.{OUTPUT_DATASET}.{OUTPUT_TABLE}`').result()))[0] +written = next(iter(bigquery.Client(project=OUTPUT_PROJECT).query( + f'SELECT COUNT(*) FROM `{OUTPUT_PROJECT}.{OUTPUT_DATASET}.{OUTPUT_TABLE}`').result()))[0] # Databricks — with the confirmed SQL warehouse; wait_timeout is at most '50s', so poll import time @@ -178,25 +195,28 @@ def q(*parts): ws = WorkspaceClient() r = ws.statement_execution.execute_statement( warehouse_id=DATABRICKS_WAREHOUSE_ID, wait_timeout='50s', - statement=f'SELECT COUNT(*) FROM {q(CATALOG, OUTPUT_SCHEMA, OUTPUT_TABLE)}') + statement=f'SELECT COUNT(*) FROM {q(OUTPUT_CATALOG, OUTPUT_SCHEMA, OUTPUT_TABLE)}') while r.status.state in (StatementState.PENDING, StatementState.RUNNING): time.sleep(5) r = ws.statement_execution.get_statement(r.statement_id) assert r.status.state == StatementState.SUCCEEDED, r.status written = int(r.result.data_array[0][0]) -# Snowflake — through the same connections.toml entry +# Snowflake — through the same connections.toml entry and warehouse. Kumo keeps the table +# name's case, so quote it (kumo_finetuning_sdk.md §9.1); database and schema as in §2. import snowflake.connector -with snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION) as conn: +with snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, + warehouse=SNOWFLAKE_WAREHOUSE) as conn: + table = '"' + OUTPUT_TABLE.replace('"', '""') + '"' written = conn.cursor().execute( - f'SELECT COUNT(*) FROM {OUTPUT_DATABASE}.{OUTPUT_SCHEMA}.{OUTPUT_TABLE}').fetchone()[0] + f'SELECT COUNT(*) FROM {OUTPUT_DATABASE}.{OUTPUT_SCHEMA}.{table}').fetchone()[0] assert written == prediction_result.num_rows, (written, prediction_result.num_rows) ``` - `OUTPUT_LOCATION` for files, S3 and GCS is the confirmed destination itself. - A Databricks destination is passed to Kumo as `'__kumo__
'` and read back as - `..
`. With a cluster and no SQL warehouse, there is no read-back here: + `..
`, in its own catalog. With a cluster and no SQL warehouse, there is no read-back here: say so in the artifact and in your reply. - Report the destination in its own form, unchanged — `DB.SCHEMA.TABLE`, or the path. @@ -229,15 +249,16 @@ system; the destination may be in another system (`adapter.md` §4) and plays no keep = {ENTITY_KEY: 'ENTITY1'} if ENTITY_TIME: # the entity Table's time_column, else None keep[ENTITY_TIME] = 'TIMESTAMP' - population = ds.dataset(ENTITY_SOURCE, format=ENTITY_FORMAT).to_table(columns=list(keep)) - population = population.rename_columns([keep[c] for c in population.column_names]) - if 'TIMESTAMP' in population.column_names: # a DATE anchor reads as null - i = population.column_names.index('TIMESTAMP') - population = population.set_column(i, 'TIMESTAMP', - pc.cast(population['TIMESTAMP'], pa.timestamp('us'))) + scanner = ds.dataset(ENTITY_SOURCE, format=ENTITY_FORMAT).scanner(columns=list(keep)) + schema = pa.schema([(keep[f.name], pa.timestamp('us') if keep[f.name] == 'TIMESTAMP' else f.type) + for f in scanner.projected_schema]) # a DATE anchor reads as null: cast if '://' not in POPULATION_PATH: os.makedirs(POPULATION_PATH, exist_ok=True) # object stores need no directory - papq.write_table(population, POPULATION_PATH + 'population.parquet') + with papq.ParquetWriter(POPULATION_PATH + 'population.parquet', schema) as writer: + for batch in scanner.to_batches(): # in batches: bounded memory + writer.write_batch(pa.RecordBatch.from_arrays( + [pc.cast(batch[f.name], t.type) for f, t in zip(batch.schema, schema)], + schema=schema)) if POPULATION_PATH.startswith('s3://'): # the source's kind, in the §2 forms population_connector = kumo.S3Connector(root_dir=POPULATION_PATH) elif POPULATION_PATH.startswith('gs://'): @@ -251,14 +272,15 @@ system; the destination may be in another system (`adapter.md` §4) and plays no `ENTITY_SOURCE` is the entity table's file or directory as preflight listed it, and `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the - source's file system or bucket. + source's file system or bucket. Each run replaces its `population.parquet`, as Node 1d says. - **Snowflake source:** the §7.1 SQL, run through - `snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION)` as in the read-back above, + `snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, warehouse=SNOWFLAKE_WAREHOUSE)` + as in the read-back above, and the `PredictionTable` built from a `SnowflakeConnector` made as in §2 but scoped to the population's database and schema. - **Databricks source:** create the table through the source's SQL warehouse with the statement API, wait for it, and build the `PredictionTable` from a `DatabricksConnector` scoped to the - population's schema. A source reached through a cluster has no SQL warehouse: say at Node 1d + population's catalog and schema, `POP_CATALOG` and `POP_SCHEMA`. A source reached through a cluster has no SQL warehouse: say at Node 1d that staging is not covered there, and offer the rows whose target is NULL instead. The same `TIMESTAMP` rule applies: project `CAST(
'` and read back as - `..
`. With a cluster and no SQL warehouse, there is no read-back here: + `..
`, in its own catalog. With a cluster and no SQL warehouse, there is no read-back here: say so in the artifact and in your reply. - Report the destination in its own form, unchanged — `DB.SCHEMA.TABLE`, or the path. @@ -229,15 +249,16 @@ system; the destination may be in another system (`adapter.md` §4) and plays no keep = {ENTITY_KEY: 'ENTITY1'} if ENTITY_TIME: # the entity Table's time_column, else None keep[ENTITY_TIME] = 'TIMESTAMP' - population = ds.dataset(ENTITY_SOURCE, format=ENTITY_FORMAT).to_table(columns=list(keep)) - population = population.rename_columns([keep[c] for c in population.column_names]) - if 'TIMESTAMP' in population.column_names: # a DATE anchor reads as null - i = population.column_names.index('TIMESTAMP') - population = population.set_column(i, 'TIMESTAMP', - pc.cast(population['TIMESTAMP'], pa.timestamp('us'))) + scanner = ds.dataset(ENTITY_SOURCE, format=ENTITY_FORMAT).scanner(columns=list(keep)) + schema = pa.schema([(keep[f.name], pa.timestamp('us') if keep[f.name] == 'TIMESTAMP' else f.type) + for f in scanner.projected_schema]) # a DATE anchor reads as null: cast if '://' not in POPULATION_PATH: os.makedirs(POPULATION_PATH, exist_ok=True) # object stores need no directory - papq.write_table(population, POPULATION_PATH + 'population.parquet') + with papq.ParquetWriter(POPULATION_PATH + 'population.parquet', schema) as writer: + for batch in scanner.to_batches(): # in batches: bounded memory + writer.write_batch(pa.RecordBatch.from_arrays( + [pc.cast(batch[f.name], t.type) for f, t in zip(batch.schema, schema)], + schema=schema)) if POPULATION_PATH.startswith('s3://'): # the source's kind, in the §2 forms population_connector = kumo.S3Connector(root_dir=POPULATION_PATH) elif POPULATION_PATH.startswith('gs://'): @@ -251,14 +272,15 @@ system; the destination may be in another system (`adapter.md` §4) and plays no `ENTITY_SOURCE` is the entity table's file or directory as preflight listed it, and `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the - source's file system or bucket. + source's file system or bucket. Each run replaces its `population.parquet`, as Node 1d says. - **Snowflake source:** the §7.1 SQL, run through - `snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION)` as in the read-back above, + `snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, warehouse=SNOWFLAKE_WAREHOUSE)` + as in the read-back above, and the `PredictionTable` built from a `SnowflakeConnector` made as in §2 but scoped to the population's database and schema. - **Databricks source:** create the table through the source's SQL warehouse with the statement API, wait for it, and build the `PredictionTable` from a `DatabricksConnector` scoped to the - population's schema. A source reached through a cluster has no SQL warehouse: say at Node 1d + population's catalog and schema, `POP_CATALOG` and `POP_SCHEMA`. A source reached through a cluster has no SQL warehouse: say at Node 1d that staging is not covered there, and offer the rows whose target is NULL instead. The same `TIMESTAMP` rule applies: project `CAST(
.: declared primary key, N rows, N_DISTINCT distinct, + N_NON_NULL non-null` — and not used; go on to the next rule. If the probe cannot run, the + declared key is `[inferred, unverified]`. + 6. Otherwise preflight **inferred no primary key**. Report that as its own line and treat the table as keyless unless corrected. A keyless table may be an edge *source*, but not an `Edge.dst_table` and not the `FOR EACH` entity. Do not conclude from this failure alone that @@ -124,7 +131,7 @@ In order of preference, first match wins: unique in the rows you can see is not a declared key, and a child table may hold several rows per parent. -Rule 1, and rules 2-4 once the probe passes, establish a key and name it `[inferred]`. Rule 5 +Rules 1-4, once the probe passes, establish a key and name it `[inferred]`. Rule 5 never establishes one: it proposes a survivor as `[inferred, unverified]`. Until the user confirms it, the table is treated exactly as keyless — usable as an edge source, but not as an edge destination and not as the `FOR EACH` diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index b476f9e84..66c5fc071 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-11-9a357803`** +**`SKILL-BUILD: 2026-11-11-b73963a9-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/accept/preflight.md b/skills/kumo-data-science/accept/preflight.md index d25a7ae68..a3382bffc 100644 --- a/skills/kumo-data-science/accept/preflight.md +++ b/skills/kumo-data-science/accept/preflight.md @@ -84,7 +84,8 @@ ORDER BY TABLE_NAME, ORDINAL_POSITION; In order of preference, first match wins: 1. A declared primary key, where the source declares any (the adapter's preflight shows how to - read them; files and object stores declare none). + read them; files and object stores declare none). It is probed like rules 2-4 (under rule 5), + because a declared key is not necessarily an enforced one. 2. `_ID`, where the stem is the table name without a `_DIM` / `_PROFILE` / `_EVENTS` suffix, singular or plural (`CUSTOMER_ID` in `CUSTOMERS`, `PAGE_VIEW_ID` in `PAGE_VIEW_EVENTS`). 3. A column named `ROW_ID`, `ID`, or `EVENT_ID`. @@ -116,6 +117,12 @@ In order of preference, first match wins: fails it is a link, not a key (`CUSTOMER_ID` in `CUSTOMER_EVENTS`); go on to the next rule. If the probe cannot run, the rule's pick is `[inferred, unverified]`, as here. + Run it on a rule 1 key as well. Snowflake, BigQuery and Databricks record a declared primary + key without enforcing it, so declared does not mean unique. A declared key that fails is + reported with its counts — `
.: declared primary key, N rows, N_DISTINCT distinct, + N_NON_NULL non-null` — and not used; go on to the next rule. If the probe cannot run, the + declared key is `[inferred, unverified]`. + 6. Otherwise preflight **inferred no primary key**. Report that as its own line and treat the table as keyless unless corrected. A keyless table may be an edge *source*, but not an `Edge.dst_table` and not the `FOR EACH` entity. Do not conclude from this failure alone that @@ -124,7 +131,7 @@ In order of preference, first match wins: unique in the rows you can see is not a declared key, and a child table may hold several rows per parent. -Rule 1, and rules 2-4 once the probe passes, establish a key and name it `[inferred]`. Rule 5 +Rules 1-4, once the probe passes, establish a key and name it `[inferred]`. Rule 5 never establishes one: it proposes a survivor as `[inferred, unverified]`. Until the user confirms it, the table is treated exactly as keyless — usable as an edge source, but not as an edge destination and not as the `FOR EACH` From d9d0ab883ade1fbff75c6fe2ee30a1c0ec9e580d Mon Sep 17 00:00:00 2001 From: igutierrez Date: Thu, 12 Nov 2026 10:37:13 -0600 Subject: [PATCH 12/35] kumo-graph-visualization: never overwrite a file with a default export Graph.visualize() overwrites an existing path (kumo.engine.graph). A script's diagram takes a new name beside .graph.svg when that one exists, the Markdown schema goes to a fresh temporary directory, and an export the user did not name a path for gets one that does not exist yet. Each reports the path it wrote. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- .../skills/kumo-graph-visualization/SKILL.md | 22 ++++++++++++++----- skills/kumo-data-science/SKILL.md | 2 +- .../kumo-graph-visualization/SKILL.md | 22 ++++++++++++++----- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/agent-skills/skills/kumo-graph-visualization/SKILL.md b/agent-skills/skills/kumo-graph-visualization/SKILL.md index abfc0b4a8..422f2722a 100644 --- a/agent-skills/skills/kumo-graph-visualization/SKILL.md +++ b/agent-skills/skills/kumo-graph-visualization/SKILL.md @@ -32,13 +32,19 @@ Continue the workflow normally; pause for visual review only if the user asks. ## Generated SDK scripts Save the diagram next to the artifact, named after it (`churn_30d_train.graph.svg` for -`churn_30d_train.py`), before training, and report its path. A fixed name such as `graph.svg` -could replace an unrelated file; the artifact's own name is only ever its own: +`churn_30d_train.py`), before training, and report its path. `visualize` overwrites an existing +file, so when that name is taken — by an earlier run or anything else — write a new file beside +it instead: ```python +import tempfile from pathlib import Path graph_path = Path(__file__).with_suffix('.graph.svg').resolve() +if graph_path.exists(): # never replace a file: take a new name beside it + with tempfile.NamedTemporaryFile(dir=graph_path.parent, prefix=graph_path.stem + '-', + suffix='.svg', delete=False) as reserved: + graph_path = Path(reserved.name) if not hasattr(graph, 'visualize'): print('Graph preview needs a kumo-engine build including #134; upgrade when available. Continuing.') else: @@ -55,7 +61,8 @@ The viewer supports up to 30 tables and does not support self-relations. ## Requested visuals and exports Display the graph in a notebook or save the format the user requests: `.svg` for a diagram, `.pdf` for a diagram and complete schema index, or `.md` for a Mermaid diagram and schema tables. -`graph.visualize('graph.pdf', show_cols=True)` also shows ordinary columns in the diagram. +`graph.visualize(EXPORT_PATH, show_cols=True)` also shows ordinary columns in the diagram. +`EXPORT_PATH` is the path the user named. When they named none, choose one that does not exist yet, as for scripts, and report it; when the named file exists, say so and confirm before replacing it. Use `layout='standard'` or `layout='compact'` to select an arrangement; the default is `'auto'`. Export paths belong to the Python runtime; use `hosts//runtime.md` for display and file retrieval. @@ -67,10 +74,15 @@ Keep samples off unless the user requests source values. When considering which tables or relationships to change for a use case, export the graph for text inspection: ```python -_ = graph.visualize('graph.md') +import tempfile +from pathlib import Path + +schema_path = Path(tempfile.mkdtemp(prefix='kumo-graph-')) / 'graph.md' # a new directory: replaces nothing +_ = graph.visualize(schema_path) +print(f'Graph schema: {schema_path}') ``` -Read the file to inspect all declared columns, key and time roles, and relationships, including columns hidden in the diagram. +Read the file at the printed path to inspect all declared columns, key and time roles, and relationships, including columns hidden in the diagram. The assignment suppresses automatic notebook display. Use this schema alongside model evaluation results to plan graph experiments; it does not measure predictive usefulness. Markdown inspection is optional during iteration, not a prerequisite for generating or running a model. diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index 66c5fc071..cf0e7fb49 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-11-b73963a9-dirty`** +**`SKILL-BUILD: 2026-11-12-7b63c2ba-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/kumo-graph-visualization/SKILL.md b/skills/kumo-data-science/kumo-graph-visualization/SKILL.md index abfc0b4a8..422f2722a 100644 --- a/skills/kumo-data-science/kumo-graph-visualization/SKILL.md +++ b/skills/kumo-data-science/kumo-graph-visualization/SKILL.md @@ -32,13 +32,19 @@ Continue the workflow normally; pause for visual review only if the user asks. ## Generated SDK scripts Save the diagram next to the artifact, named after it (`churn_30d_train.graph.svg` for -`churn_30d_train.py`), before training, and report its path. A fixed name such as `graph.svg` -could replace an unrelated file; the artifact's own name is only ever its own: +`churn_30d_train.py`), before training, and report its path. `visualize` overwrites an existing +file, so when that name is taken — by an earlier run or anything else — write a new file beside +it instead: ```python +import tempfile from pathlib import Path graph_path = Path(__file__).with_suffix('.graph.svg').resolve() +if graph_path.exists(): # never replace a file: take a new name beside it + with tempfile.NamedTemporaryFile(dir=graph_path.parent, prefix=graph_path.stem + '-', + suffix='.svg', delete=False) as reserved: + graph_path = Path(reserved.name) if not hasattr(graph, 'visualize'): print('Graph preview needs a kumo-engine build including #134; upgrade when available. Continuing.') else: @@ -55,7 +61,8 @@ The viewer supports up to 30 tables and does not support self-relations. ## Requested visuals and exports Display the graph in a notebook or save the format the user requests: `.svg` for a diagram, `.pdf` for a diagram and complete schema index, or `.md` for a Mermaid diagram and schema tables. -`graph.visualize('graph.pdf', show_cols=True)` also shows ordinary columns in the diagram. +`graph.visualize(EXPORT_PATH, show_cols=True)` also shows ordinary columns in the diagram. +`EXPORT_PATH` is the path the user named. When they named none, choose one that does not exist yet, as for scripts, and report it; when the named file exists, say so and confirm before replacing it. Use `layout='standard'` or `layout='compact'` to select an arrangement; the default is `'auto'`. Export paths belong to the Python runtime; use `hosts//runtime.md` for display and file retrieval. @@ -67,10 +74,15 @@ Keep samples off unless the user requests source values. When considering which tables or relationships to change for a use case, export the graph for text inspection: ```python -_ = graph.visualize('graph.md') +import tempfile +from pathlib import Path + +schema_path = Path(tempfile.mkdtemp(prefix='kumo-graph-')) / 'graph.md' # a new directory: replaces nothing +_ = graph.visualize(schema_path) +print(f'Graph schema: {schema_path}') ``` -Read the file to inspect all declared columns, key and time roles, and relationships, including columns hidden in the diagram. +Read the file at the printed path to inspect all declared columns, key and time roles, and relationships, including columns hidden in the diagram. The assignment suppresses automatic notebook display. Use this schema alongside model evaluation results to plan graph experiments; it does not measure predictive usefulness. Markdown inspection is optional during iteration, not a prerequisite for generating or running a model. From 2507a7fc2c9b47432420781f5cdfb38730574397 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Thu, 12 Nov 2026 10:37:13 -0600 Subject: [PATCH 13/35] kumoai-reference: generation-plan fields and destination timestamp types - pql_writing: forecast_length goes on the TrainingTableGenerationPlan for generate_training_table, and generate_prediction_table takes trainer.training_table_generation_plan (it accepts a TrainingTableGenerationPlan, not a PredictionTableGenerationPlan). - pql_writing: train_start_offset is a TrainingTableGenerationPlan field; the model plan has none. - pql_writing: get_task_type() reads the query alone, so a temporal regression meant as a forecast still returns REGRESSION. - outputs: APPEND run columns added by hand take the destination's timestamp type: TIMESTAMP_NTZ on Snowflake and Databricks, TIMESTAMP on BigQuery (kumo_bigquery.write rejects DATETIME). Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- agent-skills/skills/kumoai-reference/references/outputs.md | 7 +++++-- .../skills/kumoai-reference/references/pql_writing.md | 6 +++--- skills/kumo-data-science/SKILL.md | 2 +- .../kumoai-reference/references/outputs.md | 7 +++++-- .../kumoai-reference/references/pql_writing.md | 6 +++--- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/agent-skills/skills/kumoai-reference/references/outputs.md b/agent-skills/skills/kumoai-reference/references/outputs.md index 2d6f57c83..6fcc5ea75 100644 --- a/agent-skills/skills/kumoai-reference/references/outputs.md +++ b/agent-skills/skills/kumoai-reference/references/outputs.md @@ -56,8 +56,11 @@ whatever `extra_fields` says (Source: shipped package The write never adds a column to an existing table. When an APPEND destination already exists, read its columns before generating the artifact. If it lacks any of these columns, a run would fail at the write, after training and scoring. Say so at the gate and let the user pick: add the -missing columns (`ALTER TABLE … ADD COLUMN`: the timestamps `TIMESTAMP_NTZ`, the IDs a string -type; earlier rows hold nulls), a new table the first run creates, or OVERWRITE. Adding columns is +missing columns (`ALTER TABLE … ADD COLUMN`, earlier rows hold nulls), a new table the first run +creates, or OVERWRITE. The IDs take a string type. The timestamps take the type the destination's +sink writes: `TIMESTAMP_NTZ` on Snowflake and Databricks, `TIMESTAMP` on BigQuery, where a +`DATETIME` column fails the APPEND (Source: shipped package (`kumo_bigquery.write`, +`kumo_databricks.write`)). Adding columns is a write, confirmed like any other. The exact table name is used; no suffix is appended and its case is kept. Read back, with the diff --git a/agent-skills/skills/kumoai-reference/references/pql_writing.md b/agent-skills/skills/kumoai-reference/references/pql_writing.md index 5518f824e..e2100560d 100644 --- a/agent-skills/skills/kumoai-reference/references/pql_writing.md +++ b/agent-skills/skills/kumoai-reference/references/pql_writing.md @@ -212,7 +212,7 @@ Kumo determines the task type automatically. `pquery.get_task_type()` returns it | `LIST_DISTINCT` over a **non-foreign-key** categorical column | Multilabel classification | | `LIST_DISTINCT` over a **foreign key** column (+ `RANK TOP K`) | Link prediction / ranking | | `LIST_DISTINCT(...) CLASSIFY` | Separate binary classification per class | -| Temporal regression + generation-plan `forecast_length` | Forecasting | +| Temporal regression meant as a forecast | Regression: `get_task_type()` reads the query alone, so it does not change when the generation plan sets `forecast_length`; that plan is what makes it a forecast (*Forecasting* below) | Static vs temporal: a query without a time window is **static** (default 80/10/10 random split, no time column required). A query aggregating over a window is **temporal** (time-based splits, anchor times sampled back through history). @@ -367,7 +367,7 @@ Produces a separate binary classification per article ID. Without `TOP K`, class ### Forecasting -Forecasting is **not a distinct PQL keyword**. Write a temporal regression query and set `forecast_length` in the training and prediction table generation plans (`TrainingTableGenerationPlan`, `PredictionTableGenerationPlan`; Source: shipped package (`kumoapi.model_plan`)), not the model plan; `forecast_length=3` on a 7-day window forecasts the next three 7-day windows. The flow is in `kumoai-reference/references/task_journeys.md` *Forecasting*. +Forecasting is **not a distinct PQL keyword**. Write a temporal regression query, set `forecast_length` on the `TrainingTableGenerationPlan` passed to `generate_training_table(snapshot, plan=...)`, and pass the fitted trainer's `trainer.training_table_generation_plan` to `generate_prediction_table(snapshot, plan=...)` — both take a `TrainingTableGenerationPlan`, not a `PredictionTableGenerationPlan`, and never the model plan (Source: shipped package (`kumoapi.model_plan`, `kumo.engine.pquery.predictive_query`)); `forecast_length=3` on a 7-day window forecasts the next three 7-day windows. The flow is in `kumoai-reference/references/task_journeys.md` *Forecasting*. ```text PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 7, days) @@ -429,7 +429,7 @@ Source: shipped package (`kumo_core.model_plan`) 8. **`LAST`/`FIRST` cannot be applied to foreign key columns.** Use `LIST_DISTINCT` + `RANK` for FK targets. 9. **`LIST_DISTINCT` over a non-FK column silently becomes multilabel classification, not ranking.** Check `get_task_type()`. 10. **`LIST_DISTINCT` beyond ~1000 distinct values is not recommended.** -11. **Horizon vs. split health:** too large a window leaves fewer than 3 anchor times; too small a window on a long history generates too many examples (mitigate with model-plan `train_start_offset`) and, for link prediction, too few positives. +11. **Horizon vs. split health:** too large a window leaves fewer than 3 anchor times; too small a window on a long history generates too many examples (mitigate with `train_start_offset` on the `TrainingTableGenerationPlan` passed to `generate_training_table(snapshot, plan=...)`; the model plan has no such field) and, for link prediction, too few positives. 12. **Always validate before generating tables:** `pquery.validate()` for syntax, `pquery.get_task_type()` for semantics. Required — see "Required pre-flight" under SDK usage. 13. **Inline `WHERE` vs trailing `WHERE` are not interchangeable.** A row-level condition on the predicted event goes inside the aggregation; an entity-eligibility condition goes in the trailing `WHERE`. Swapping them produces a query that validates and trains on the wrong question. diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index cf0e7fb49..31fd18d8e 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-12-7b63c2ba-dirty`** +**`SKILL-BUILD: 2026-11-12-d9d0ab88-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/kumoai-reference/references/outputs.md b/skills/kumo-data-science/kumoai-reference/references/outputs.md index 2d6f57c83..6fcc5ea75 100644 --- a/skills/kumo-data-science/kumoai-reference/references/outputs.md +++ b/skills/kumo-data-science/kumoai-reference/references/outputs.md @@ -56,8 +56,11 @@ whatever `extra_fields` says (Source: shipped package The write never adds a column to an existing table. When an APPEND destination already exists, read its columns before generating the artifact. If it lacks any of these columns, a run would fail at the write, after training and scoring. Say so at the gate and let the user pick: add the -missing columns (`ALTER TABLE … ADD COLUMN`: the timestamps `TIMESTAMP_NTZ`, the IDs a string -type; earlier rows hold nulls), a new table the first run creates, or OVERWRITE. Adding columns is +missing columns (`ALTER TABLE … ADD COLUMN`, earlier rows hold nulls), a new table the first run +creates, or OVERWRITE. The IDs take a string type. The timestamps take the type the destination's +sink writes: `TIMESTAMP_NTZ` on Snowflake and Databricks, `TIMESTAMP` on BigQuery, where a +`DATETIME` column fails the APPEND (Source: shipped package (`kumo_bigquery.write`, +`kumo_databricks.write`)). Adding columns is a write, confirmed like any other. The exact table name is used; no suffix is appended and its case is kept. Read back, with the diff --git a/skills/kumo-data-science/kumoai-reference/references/pql_writing.md b/skills/kumo-data-science/kumoai-reference/references/pql_writing.md index 5518f824e..e2100560d 100644 --- a/skills/kumo-data-science/kumoai-reference/references/pql_writing.md +++ b/skills/kumo-data-science/kumoai-reference/references/pql_writing.md @@ -212,7 +212,7 @@ Kumo determines the task type automatically. `pquery.get_task_type()` returns it | `LIST_DISTINCT` over a **non-foreign-key** categorical column | Multilabel classification | | `LIST_DISTINCT` over a **foreign key** column (+ `RANK TOP K`) | Link prediction / ranking | | `LIST_DISTINCT(...) CLASSIFY` | Separate binary classification per class | -| Temporal regression + generation-plan `forecast_length` | Forecasting | +| Temporal regression meant as a forecast | Regression: `get_task_type()` reads the query alone, so it does not change when the generation plan sets `forecast_length`; that plan is what makes it a forecast (*Forecasting* below) | Static vs temporal: a query without a time window is **static** (default 80/10/10 random split, no time column required). A query aggregating over a window is **temporal** (time-based splits, anchor times sampled back through history). @@ -367,7 +367,7 @@ Produces a separate binary classification per article ID. Without `TOP K`, class ### Forecasting -Forecasting is **not a distinct PQL keyword**. Write a temporal regression query and set `forecast_length` in the training and prediction table generation plans (`TrainingTableGenerationPlan`, `PredictionTableGenerationPlan`; Source: shipped package (`kumoapi.model_plan`)), not the model plan; `forecast_length=3` on a 7-day window forecasts the next three 7-day windows. The flow is in `kumoai-reference/references/task_journeys.md` *Forecasting*. +Forecasting is **not a distinct PQL keyword**. Write a temporal regression query, set `forecast_length` on the `TrainingTableGenerationPlan` passed to `generate_training_table(snapshot, plan=...)`, and pass the fitted trainer's `trainer.training_table_generation_plan` to `generate_prediction_table(snapshot, plan=...)` — both take a `TrainingTableGenerationPlan`, not a `PredictionTableGenerationPlan`, and never the model plan (Source: shipped package (`kumoapi.model_plan`, `kumo.engine.pquery.predictive_query`)); `forecast_length=3` on a 7-day window forecasts the next three 7-day windows. The flow is in `kumoai-reference/references/task_journeys.md` *Forecasting*. ```text PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 7, days) @@ -429,7 +429,7 @@ Source: shipped package (`kumo_core.model_plan`) 8. **`LAST`/`FIRST` cannot be applied to foreign key columns.** Use `LIST_DISTINCT` + `RANK` for FK targets. 9. **`LIST_DISTINCT` over a non-FK column silently becomes multilabel classification, not ranking.** Check `get_task_type()`. 10. **`LIST_DISTINCT` beyond ~1000 distinct values is not recommended.** -11. **Horizon vs. split health:** too large a window leaves fewer than 3 anchor times; too small a window on a long history generates too many examples (mitigate with model-plan `train_start_offset`) and, for link prediction, too few positives. +11. **Horizon vs. split health:** too large a window leaves fewer than 3 anchor times; too small a window on a long history generates too many examples (mitigate with `train_start_offset` on the `TrainingTableGenerationPlan` passed to `generate_training_table(snapshot, plan=...)`; the model plan has no such field) and, for link prediction, too few positives. 12. **Always validate before generating tables:** `pquery.validate()` for syntax, `pquery.get_task_type()` for semantics. Required — see "Required pre-flight" under SDK usage. 13. **Inline `WHERE` vs trailing `WHERE` are not interchangeable.** A row-level condition on the predicted event goes inside the aggregation; an entity-eligibility condition goes in the trailing `WHERE`. Swapping them produces a query that validates and trains on the wrong question. From 21c1f7ab783fcd0f89797347a62194e2f409bd93 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Thu, 12 Nov 2026 10:37:33 -0600 Subject: [PATCH 14/35] kumo-data-science: bind population replacement to its name; stage it safely; check APPEND by delta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Node 1d: the replacement approval covers the confirmed population name only. The config cell carries it as POPULATION_CONFIRMED, and every staging recipe (kumo_finetuning_sdk.md §7.1, generic runtime §3.1, Genie runtime §4.1) stops before CREATE OR REPLACE or the file write when the name differs. - Node 1d: the default population beside a path destination is the sibling _population/, not a directory inside the destination, which must stay empty. - Generic runtime §3.1, file staging: the scan and the writer take the §2 S3FileSystem when the user gave an explicit S3Identity; a source timestamp keeps its unit (a safe cast of sub-microsecond timestamp[ns] to us raises), only a DATE is cast; rows go to population.parquet.partial and replace population.parquet only after every batch is written. - Read-back after APPEND (generic runtime §3, kumo_finetuning_sdk.md §9.1 and checklist): compare the rows added since a count taken before predict, 0 when the table does not exist yet, as outputs.md says. - kumo_finetuning_sdk.md §10: the get_active_session() example is scoped to hosts that provide a session; other hosts drop it and use their runtime.md connect step and SQL read-back. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- .../skills/hosts/databricks-genie/runtime.md | 6 +- agent-skills/skills/hosts/generic/runtime.md | 55 ++++++++++++++++--- .../skills/kumo-data-science/SKILL.md | 13 +++-- .../references/kumo_finetuning_sdk.md | 28 +++++++--- skills/kumo-data-science/SKILL.md | 15 +++-- .../hosts/generic/runtime.md | 55 ++++++++++++++++--- .../references/kumo_finetuning_sdk.md | 28 +++++++--- 7 files changed, 158 insertions(+), 42 deletions(-) diff --git a/agent-skills/skills/hosts/databricks-genie/runtime.md b/agent-skills/skills/hosts/databricks-genie/runtime.md index f86091c00..742919cb9 100644 --- a/agent-skills/skills/hosts/databricks-genie/runtime.md +++ b/agent-skills/skills/hosts/databricks-genie/runtime.md @@ -172,6 +172,8 @@ without it, even for a static query, whatever the §7.1 Snowflake example shows, anchor as null. Without a time column, select `ENTITY1` alone. ```python +if f'{POP_CATALOG}.{POP_SCHEMA}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d + raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') run_sql(f'CREATE OR REPLACE TABLE {q(POP_CATALOG, POP_SCHEMA, POP_TABLE)} AS ' f'SELECT {q(ENTITY_KEY)} AS ENTITY1, CAST({q(ENTITY_TIME)} AS TIMESTAMP) AS TIMESTAMP ' f'FROM {q(CATALOG, SCHEMA, ENTITY_TABLE)}') @@ -182,7 +184,9 @@ prediction_table = kumo.PredictionTable.from_source_table( ``` `ENTITY_TIME` is the entity `Table`'s `time_column`; `POP_CATALOG` and `POP_SCHEMA` are the -population's confirmed location, which need not be the source's. +population's confirmed location, which need not be the source's. `POPULATION_CONFIRMED` is the +population exactly as Node 1d confirmed it (`kumoai-reference/references/kumo_finetuning_sdk.md` +§7.1). **A population the user supplies** — a table of exact scoring rows, or curated labels, weights and splits (`kumo-custom-populations/SKILL.md`) — is read through a `DatabricksConnector` on that diff --git a/agent-skills/skills/hosts/generic/runtime.md b/agent-skills/skills/hosts/generic/runtime.md index cea0c6b06..75b2f4e4d 100644 --- a/agent-skills/skills/hosts/generic/runtime.md +++ b/agent-skills/skills/hosts/generic/runtime.md @@ -28,7 +28,9 @@ still waits for an explicit request. ## 1. Config cell The first code cell holds the imports and every location as a named constant: the confirmed -source, the confirmed destination, and any population table — each written verbatim. +source, the confirmed destination, and any population table — each written verbatim. A +population also gets `POPULATION_CONFIRMED`, its name exactly as Node 1d confirmed it, which the +staging step checks before replacing anything (§3.1). Values only the runtime knows — a Snowflake warehouse or `connections.toml` entry, a Databricks SQL warehouse or cluster — are constants here too. S3, GCS and BigQuery have none: the ambient @@ -128,6 +130,9 @@ written = ds.dataset(OUTPUT_LOCATION.removeprefix('s3://'), format=OUTPUT_FORMAT filesystem=s3).count_rows() ``` +Without an explicit identity, emit `s3 = None`: §3.1's population staging then lets Arrow resolve +the ambient credentials itself. + The Snowflake factory `from_spcs_env` and Snowflake's notebook secret store belong to Snowflake's own runtime. Emit them only when the user says the code runs there. @@ -211,9 +216,15 @@ with snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, written = conn.cursor().execute( f'SELECT COUNT(*) FROM {OUTPUT_DATABASE}.{OUTPUT_SCHEMA}.{table}').fetchone()[0] -assert written == prediction_result.num_rows, (written, prediction_result.num_rows) +assert written - before == prediction_result.num_rows, (written, before, prediction_result.num_rows) ``` +`before` is 0 for OVERWRITE. For APPEND (`kumoai-reference/references/outputs.md`) it is the same +count taken before `predict` — 0 when the destination does not exist yet, since the first APPEND +creates it — so the check compares the rows this run added. The delta holds only while nothing +else writes to the destination; when the user says others do, confirm a run-specific destination +instead. + - `OUTPUT_LOCATION` for files, S3 and GCS is the confirmed destination itself. - A Databricks destination is passed to Kumo as `'__kumo__
'` and read back as `..
`, in its own catalog. With a cluster and no SQL warehouse, there is no read-back here: @@ -244,21 +255,39 @@ system; the destination may be in another system (`adapter.md` §4) and plays no import pyarrow as pa import pyarrow.compute as pc import pyarrow.dataset as ds + import pyarrow.fs import pyarrow.parquet as papq + def arrow_fs(uri): + """Arrow's filesystem and path for uri; S3 with an explicit S3Identity takes §2's `s3`.""" + if uri.startswith('s3://') and s3 is not None: # s3 = None under ambient credentials + return s3, uri.removeprefix('s3://') + return pyarrow.fs.FileSystem.from_uri(uri if '://' in uri else os.path.abspath(uri)) + + if POPULATION_PATH != POPULATION_CONFIRMED: # Node 1d approved replacing this path only + raise RuntimeError('POPULATION_PATH changed after Node 1d: confirm the new path first.') keep = {ENTITY_KEY: 'ENTITY1'} if ENTITY_TIME: # the entity Table's time_column, else None keep[ENTITY_TIME] = 'TIMESTAMP' - scanner = ds.dataset(ENTITY_SOURCE, format=ENTITY_FORMAT).scanner(columns=list(keep)) - schema = pa.schema([(keep[f.name], pa.timestamp('us') if keep[f.name] == 'TIMESTAMP' else f.type) - for f in scanner.projected_schema]) # a DATE anchor reads as null: cast - if '://' not in POPULATION_PATH: - os.makedirs(POPULATION_PATH, exist_ok=True) # object stores need no directory - with papq.ParquetWriter(POPULATION_PATH + 'population.parquet', schema) as writer: + source_fs, source_path = arrow_fs(ENTITY_SOURCE) + scanner = ds.dataset(source_path, format=ENTITY_FORMAT, + filesystem=source_fs).scanner(columns=list(keep)) + + def anchor_type(t): # a timestamp keeps its unit; a DATE anchor reads as null: cast it + return t if pa.types.is_timestamp(t) else pa.timestamp('us') + + schema = pa.schema([(keep[f.name], anchor_type(f.type) if keep[f.name] == 'TIMESTAMP' else f.type) + for f in scanner.projected_schema]) + pop_fs, pop_dir = arrow_fs(POPULATION_PATH) + if isinstance(pop_fs, pyarrow.fs.LocalFileSystem): + pop_fs.create_dir(pop_dir) # object stores need no directory + final = pop_dir.rstrip('/') + '/population.parquet' + with papq.ParquetWriter(final + '.partial', schema, filesystem=pop_fs) as writer: for batch in scanner.to_batches(): # in batches: bounded memory writer.write_batch(pa.RecordBatch.from_arrays( [pc.cast(batch[f.name], t.type) for f, t in zip(batch.schema, schema)], schema=schema)) + pop_fs.move(final + '.partial', final) # replace only once every batch is in if POPULATION_PATH.startswith('s3://'): # the source's kind, in the §2 forms population_connector = kumo.S3Connector(root_dir=POPULATION_PATH) elif POPULATION_PATH.startswith('gs://'): @@ -272,7 +301,11 @@ system; the destination may be in another system (`adapter.md` §4) and plays no `ENTITY_SOURCE` is the entity table's file or directory as preflight listed it, and `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the - source's file system or bucket. Each run replaces its `population.parquet`, as Node 1d says. + source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed + it, both in the config cell. Each run replaces its `population.parquet`, as Node 1d says. The + rows go to `population.parquet.partial` first, so a scan or cast that fails leaves the previous + population in place. A source timestamp keeps its own unit: a `timestamp[ns]` with + sub-microsecond values would fail a safe cast to microseconds. - **Snowflake source:** the §7.1 SQL, run through `snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, warehouse=SNOWFLAKE_WAREHOUSE)` as in the read-back above, @@ -294,6 +327,8 @@ system; the destination may be in another system (`adapter.md` §4) and plays no """`catalog`.`schema`.`table`: Unity Catalog names may contain hyphens.""" return '.'.join('`' + p.replace('`', '``') + '`' for p in parts) + if f'{POP_CATALOG}.{POP_SCHEMA}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d + raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') ws = WorkspaceClient() time_col = f', CAST({q(ENTITY_TIME)} AS TIMESTAMP) AS TIMESTAMP' if ENTITY_TIME else '' r = ws.statement_execution.execute_statement( @@ -317,6 +352,8 @@ system; the destination may be in another system (`adapter.md` §4) and plays no ```python from google.cloud import bigquery + if f'{POP_PROJECT}.{POP_DATASET}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d + raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') time_col = f', CAST({ENTITY_TIME} AS TIMESTAMP) AS `TIMESTAMP`' if ENTITY_TIME else '' bigquery.Client(project=POP_PROJECT).query( f'CREATE OR REPLACE TABLE `{POP_PROJECT}.{POP_DATASET}.{POP_TABLE}` AS ' diff --git a/agent-skills/skills/kumo-data-science/SKILL.md b/agent-skills/skills/kumo-data-science/SKILL.md index 47c0bc515..fdc1ea534 100644 --- a/agent-skills/skills/kumo-data-science/SKILL.md +++ b/agent-skills/skills/kumo-data-science/SKILL.md @@ -240,9 +240,12 @@ generated before the task is confirmed: target is NULL (`kumoai-reference/references/kumo_finetuning_sdk.md` §7.1). If the user wants rows that already have a target scored (for example "every order"), the notebook stages an explicit population table. That is a write, and it goes in the source's system, so name it here: next to the confirmed -destination by default (`_POPULATION`) when the destination is in the source's -system; otherwise, or when there is no destination, ask for a schema or path in the source's -system. Never stage it in the source schema. +destination by default when the destination is in the source's system — the table +`_POPULATION`, or for a path the sibling directory +`_population/` (`s3://b/scores/` → +`s3://b/scores_population/`), never inside the destination, which must stay empty; otherwise, or +when there is no destination, ask for a schema or path in the source's system. Never stage it in +the source schema. **Name what a run replaces.** The notebook writes predictions with `OVERWRITE` and stages a population with `CREATE OR REPLACE` (a file: overwritten), on every run. Check from the catalog @@ -250,7 +253,9 @@ whether the destination and the population already exist, and say so on their li ` (exists: replaced on every run)` — so confirming them approves the replacement. When the user does not want an existing table replaced, ask for another name, or for a destination, APPEND (`kumoai-reference/references/outputs.md`). If the check cannot run, -say that instead. +say that instead. The approval covers the confirmed population name only: the notebook carries it +as `POPULATION_CONFIRMED` and stops before staging if the name it would replace differs +(`kumoai-reference/references/kumo_finetuning_sdk.md` §7.1). | Condition | Transition | |---|---| diff --git a/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md b/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md index aac3eeeb0..c5aaf7419 100644 --- a/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md +++ b/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md @@ -470,6 +470,8 @@ says how a population is staged, or that it is not covered there: `ENTITY1` in the confirmed population table and pass it explicitly: ```python +if f'{POP_DATABASE}.{POP_SCHEMA}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d + raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') session.sql( # the session from §3.3 f'CREATE OR REPLACE TABLE {POP_DATABASE}.{POP_SCHEMA}.{POP_TABLE} AS ' f'SELECT "{ENTITY_KEY}" AS ENTITY1 FROM {SOURCE_DATABASE}.{SOURCE_SCHEMA}."{ENTITY_TABLE}"' @@ -482,7 +484,10 @@ prediction_table = kumo.PredictionTable.from_source_table( ) ``` -- `CREATE OR REPLACE` replaces an existing table; Node 1d names that before it is confirmed. +- `CREATE OR REPLACE` replaces an existing table; Node 1d names that before it is confirmed, + and the approval covers that one name. The config cell carries it verbatim as + `POPULATION_CONFIRMED`, and every staging recipe stops before writing when the name it would + replace differs — an edit after confirmation needs a new confirmation, not a silent replace. - Stage the population only where Node 1d confirmed it — never in the source schema. Write `POP_TABLE` in upper case, so the unquoted `CREATE` and the connector lookup agree. - Never change the query, the entity or the target to get rows. An empty population is a fact @@ -623,9 +628,14 @@ def quoted(name): return '"' + name.replace('"', '""') + '"' destination = session.table([OUTPUT_DATABASE, OUTPUT_SCHEMA, quoted(output_table)]) -assert destination.count() == prediction_result.num_rows +assert destination.count() - before == prediction_result.num_rows ``` +`before` is 0 for `OVERWRITE`. For `APPEND` it is the same count taken before `predict` — 0 when +the table does not exist yet, since the first APPEND creates it — so the check compares the rows +this run added. The delta holds only while nothing else writes to the table +(`kumoai-reference/references/outputs.md`). + Scope the output connector to the destination database and schema; `table_name` is a bare name. Always read the table back — it is the only proof the write-back worked. `session` is the one taken in the first cell (§3.3); calling `get_active_session()` here, after Kumo has written, @@ -653,8 +663,10 @@ the write, as the host's `runtime.md` says (§9.1 for Snowflake). ## 10. Complete verified example -On Snowflake. On any other connector, replace steps 1, 7 and 8 with the host `runtime.md`'s -connect step, the §9.2 destination and its read-back; nothing else changes. +On Snowflake, in a host whose `runtime.md` provides an active Snowpark session (§3.3). On any +other host — Snowflake reached from outside it included — drop the `get_active_session()` line and +replace steps 1, 7 and 8 with that host `runtime.md`'s connect step, the §9.2 destination and its +SQL read-back; nothing else changes. Self-contained: `OUTPUT_DATABASE`, `OUTPUT_SCHEMA` and `OUTPUT_TABLE` are placeholders for the confirmed output destination (§12). @@ -672,7 +684,7 @@ from snowflake.snowpark.context import get_active_session OUTPUT_DATABASE, OUTPUT_SCHEMA, OUTPUT_TABLE = '', '', '
' # confirmed destination -session = get_active_session() +session = get_active_session() # §3.3: only where the host provides the session # 1. Scratch directory, then connect (inside SPCS; elsewhere use the §3 order). No credentials in the notebook. kumo.configure(scratch_dir=Path('/tmp/kumo-run') / 'scratch') @@ -856,10 +868,12 @@ targets a superseded build and needs rewriting against §10 — do not copy its - [ ] No code cell writes anywhere unless a destination was confirmed; an example of how to add one later is markdown text, not a code cell. - [ ] Smoke-test sizing called out as such. -- [ ] `write_mode` and any population replacement as Node 1d confirmed them (§9, §7.1). +- [ ] `write_mode` and any population replacement as Node 1d confirmed them (§9, §7.1); the + staging cell stops when the population name differs from `POPULATION_CONFIRMED`. - [ ] Destination in its connector's form (§9.2), reported with its exact name — no `_predictions` suffix. -- [ ] Written destination read back and row count compared to `prediction_result.num_rows`. +- [ ] Written destination read back and row count compared to `prediction_result.num_rows` — + for APPEND, the count added since before `predict` (§9.1). - [ ] Success judged by blocking calls returning without raising — not by a status value. --- diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index 31fd18d8e..8494a6a48 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-12-d9d0ab88-dirty`** +**`SKILL-BUILD: 2026-11-12-2507a7fc-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. @@ -240,9 +240,12 @@ generated before the task is confirmed: target is NULL (`kumoai-reference/references/kumo_finetuning_sdk.md` §7.1). If the user wants rows that already have a target scored (for example "every order"), the notebook stages an explicit population table. That is a write, and it goes in the source's system, so name it here: next to the confirmed -destination by default (`_POPULATION`) when the destination is in the source's -system; otherwise, or when there is no destination, ask for a schema or path in the source's -system. Never stage it in the source schema. +destination by default when the destination is in the source's system — the table +`_POPULATION`, or for a path the sibling directory +`_population/` (`s3://b/scores/` → +`s3://b/scores_population/`), never inside the destination, which must stay empty; otherwise, or +when there is no destination, ask for a schema or path in the source's system. Never stage it in +the source schema. **Name what a run replaces.** The notebook writes predictions with `OVERWRITE` and stages a population with `CREATE OR REPLACE` (a file: overwritten), on every run. Check from the catalog @@ -250,7 +253,9 @@ whether the destination and the population already exist, and say so on their li ` (exists: replaced on every run)` — so confirming them approves the replacement. When the user does not want an existing table replaced, ask for another name, or for a destination, APPEND (`kumoai-reference/references/outputs.md`). If the check cannot run, -say that instead. +say that instead. The approval covers the confirmed population name only: the notebook carries it +as `POPULATION_CONFIRMED` and stops before staging if the name it would replace differs +(`kumoai-reference/references/kumo_finetuning_sdk.md` §7.1). | Condition | Transition | |---|---| diff --git a/skills/kumo-data-science/hosts/generic/runtime.md b/skills/kumo-data-science/hosts/generic/runtime.md index cea0c6b06..75b2f4e4d 100644 --- a/skills/kumo-data-science/hosts/generic/runtime.md +++ b/skills/kumo-data-science/hosts/generic/runtime.md @@ -28,7 +28,9 @@ still waits for an explicit request. ## 1. Config cell The first code cell holds the imports and every location as a named constant: the confirmed -source, the confirmed destination, and any population table — each written verbatim. +source, the confirmed destination, and any population table — each written verbatim. A +population also gets `POPULATION_CONFIRMED`, its name exactly as Node 1d confirmed it, which the +staging step checks before replacing anything (§3.1). Values only the runtime knows — a Snowflake warehouse or `connections.toml` entry, a Databricks SQL warehouse or cluster — are constants here too. S3, GCS and BigQuery have none: the ambient @@ -128,6 +130,9 @@ written = ds.dataset(OUTPUT_LOCATION.removeprefix('s3://'), format=OUTPUT_FORMAT filesystem=s3).count_rows() ``` +Without an explicit identity, emit `s3 = None`: §3.1's population staging then lets Arrow resolve +the ambient credentials itself. + The Snowflake factory `from_spcs_env` and Snowflake's notebook secret store belong to Snowflake's own runtime. Emit them only when the user says the code runs there. @@ -211,9 +216,15 @@ with snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, written = conn.cursor().execute( f'SELECT COUNT(*) FROM {OUTPUT_DATABASE}.{OUTPUT_SCHEMA}.{table}').fetchone()[0] -assert written == prediction_result.num_rows, (written, prediction_result.num_rows) +assert written - before == prediction_result.num_rows, (written, before, prediction_result.num_rows) ``` +`before` is 0 for OVERWRITE. For APPEND (`kumoai-reference/references/outputs.md`) it is the same +count taken before `predict` — 0 when the destination does not exist yet, since the first APPEND +creates it — so the check compares the rows this run added. The delta holds only while nothing +else writes to the destination; when the user says others do, confirm a run-specific destination +instead. + - `OUTPUT_LOCATION` for files, S3 and GCS is the confirmed destination itself. - A Databricks destination is passed to Kumo as `'__kumo__
'` and read back as `..
`, in its own catalog. With a cluster and no SQL warehouse, there is no read-back here: @@ -244,21 +255,39 @@ system; the destination may be in another system (`adapter.md` §4) and plays no import pyarrow as pa import pyarrow.compute as pc import pyarrow.dataset as ds + import pyarrow.fs import pyarrow.parquet as papq + def arrow_fs(uri): + """Arrow's filesystem and path for uri; S3 with an explicit S3Identity takes §2's `s3`.""" + if uri.startswith('s3://') and s3 is not None: # s3 = None under ambient credentials + return s3, uri.removeprefix('s3://') + return pyarrow.fs.FileSystem.from_uri(uri if '://' in uri else os.path.abspath(uri)) + + if POPULATION_PATH != POPULATION_CONFIRMED: # Node 1d approved replacing this path only + raise RuntimeError('POPULATION_PATH changed after Node 1d: confirm the new path first.') keep = {ENTITY_KEY: 'ENTITY1'} if ENTITY_TIME: # the entity Table's time_column, else None keep[ENTITY_TIME] = 'TIMESTAMP' - scanner = ds.dataset(ENTITY_SOURCE, format=ENTITY_FORMAT).scanner(columns=list(keep)) - schema = pa.schema([(keep[f.name], pa.timestamp('us') if keep[f.name] == 'TIMESTAMP' else f.type) - for f in scanner.projected_schema]) # a DATE anchor reads as null: cast - if '://' not in POPULATION_PATH: - os.makedirs(POPULATION_PATH, exist_ok=True) # object stores need no directory - with papq.ParquetWriter(POPULATION_PATH + 'population.parquet', schema) as writer: + source_fs, source_path = arrow_fs(ENTITY_SOURCE) + scanner = ds.dataset(source_path, format=ENTITY_FORMAT, + filesystem=source_fs).scanner(columns=list(keep)) + + def anchor_type(t): # a timestamp keeps its unit; a DATE anchor reads as null: cast it + return t if pa.types.is_timestamp(t) else pa.timestamp('us') + + schema = pa.schema([(keep[f.name], anchor_type(f.type) if keep[f.name] == 'TIMESTAMP' else f.type) + for f in scanner.projected_schema]) + pop_fs, pop_dir = arrow_fs(POPULATION_PATH) + if isinstance(pop_fs, pyarrow.fs.LocalFileSystem): + pop_fs.create_dir(pop_dir) # object stores need no directory + final = pop_dir.rstrip('/') + '/population.parquet' + with papq.ParquetWriter(final + '.partial', schema, filesystem=pop_fs) as writer: for batch in scanner.to_batches(): # in batches: bounded memory writer.write_batch(pa.RecordBatch.from_arrays( [pc.cast(batch[f.name], t.type) for f, t in zip(batch.schema, schema)], schema=schema)) + pop_fs.move(final + '.partial', final) # replace only once every batch is in if POPULATION_PATH.startswith('s3://'): # the source's kind, in the §2 forms population_connector = kumo.S3Connector(root_dir=POPULATION_PATH) elif POPULATION_PATH.startswith('gs://'): @@ -272,7 +301,11 @@ system; the destination may be in another system (`adapter.md` §4) and plays no `ENTITY_SOURCE` is the entity table's file or directory as preflight listed it, and `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the - source's file system or bucket. Each run replaces its `population.parquet`, as Node 1d says. + source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed + it, both in the config cell. Each run replaces its `population.parquet`, as Node 1d says. The + rows go to `population.parquet.partial` first, so a scan or cast that fails leaves the previous + population in place. A source timestamp keeps its own unit: a `timestamp[ns]` with + sub-microsecond values would fail a safe cast to microseconds. - **Snowflake source:** the §7.1 SQL, run through `snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, warehouse=SNOWFLAKE_WAREHOUSE)` as in the read-back above, @@ -294,6 +327,8 @@ system; the destination may be in another system (`adapter.md` §4) and plays no """`catalog`.`schema`.`table`: Unity Catalog names may contain hyphens.""" return '.'.join('`' + p.replace('`', '``') + '`' for p in parts) + if f'{POP_CATALOG}.{POP_SCHEMA}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d + raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') ws = WorkspaceClient() time_col = f', CAST({q(ENTITY_TIME)} AS TIMESTAMP) AS TIMESTAMP' if ENTITY_TIME else '' r = ws.statement_execution.execute_statement( @@ -317,6 +352,8 @@ system; the destination may be in another system (`adapter.md` §4) and plays no ```python from google.cloud import bigquery + if f'{POP_PROJECT}.{POP_DATASET}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d + raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') time_col = f', CAST({ENTITY_TIME} AS TIMESTAMP) AS `TIMESTAMP`' if ENTITY_TIME else '' bigquery.Client(project=POP_PROJECT).query( f'CREATE OR REPLACE TABLE `{POP_PROJECT}.{POP_DATASET}.{POP_TABLE}` AS ' diff --git a/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md b/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md index aac3eeeb0..c5aaf7419 100644 --- a/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md +++ b/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md @@ -470,6 +470,8 @@ says how a population is staged, or that it is not covered there: `ENTITY1` in the confirmed population table and pass it explicitly: ```python +if f'{POP_DATABASE}.{POP_SCHEMA}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d + raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') session.sql( # the session from §3.3 f'CREATE OR REPLACE TABLE {POP_DATABASE}.{POP_SCHEMA}.{POP_TABLE} AS ' f'SELECT "{ENTITY_KEY}" AS ENTITY1 FROM {SOURCE_DATABASE}.{SOURCE_SCHEMA}."{ENTITY_TABLE}"' @@ -482,7 +484,10 @@ prediction_table = kumo.PredictionTable.from_source_table( ) ``` -- `CREATE OR REPLACE` replaces an existing table; Node 1d names that before it is confirmed. +- `CREATE OR REPLACE` replaces an existing table; Node 1d names that before it is confirmed, + and the approval covers that one name. The config cell carries it verbatim as + `POPULATION_CONFIRMED`, and every staging recipe stops before writing when the name it would + replace differs — an edit after confirmation needs a new confirmation, not a silent replace. - Stage the population only where Node 1d confirmed it — never in the source schema. Write `POP_TABLE` in upper case, so the unquoted `CREATE` and the connector lookup agree. - Never change the query, the entity or the target to get rows. An empty population is a fact @@ -623,9 +628,14 @@ def quoted(name): return '"' + name.replace('"', '""') + '"' destination = session.table([OUTPUT_DATABASE, OUTPUT_SCHEMA, quoted(output_table)]) -assert destination.count() == prediction_result.num_rows +assert destination.count() - before == prediction_result.num_rows ``` +`before` is 0 for `OVERWRITE`. For `APPEND` it is the same count taken before `predict` — 0 when +the table does not exist yet, since the first APPEND creates it — so the check compares the rows +this run added. The delta holds only while nothing else writes to the table +(`kumoai-reference/references/outputs.md`). + Scope the output connector to the destination database and schema; `table_name` is a bare name. Always read the table back — it is the only proof the write-back worked. `session` is the one taken in the first cell (§3.3); calling `get_active_session()` here, after Kumo has written, @@ -653,8 +663,10 @@ the write, as the host's `runtime.md` says (§9.1 for Snowflake). ## 10. Complete verified example -On Snowflake. On any other connector, replace steps 1, 7 and 8 with the host `runtime.md`'s -connect step, the §9.2 destination and its read-back; nothing else changes. +On Snowflake, in a host whose `runtime.md` provides an active Snowpark session (§3.3). On any +other host — Snowflake reached from outside it included — drop the `get_active_session()` line and +replace steps 1, 7 and 8 with that host `runtime.md`'s connect step, the §9.2 destination and its +SQL read-back; nothing else changes. Self-contained: `OUTPUT_DATABASE`, `OUTPUT_SCHEMA` and `OUTPUT_TABLE` are placeholders for the confirmed output destination (§12). @@ -672,7 +684,7 @@ from snowflake.snowpark.context import get_active_session OUTPUT_DATABASE, OUTPUT_SCHEMA, OUTPUT_TABLE = '', '', '
' # confirmed destination -session = get_active_session() +session = get_active_session() # §3.3: only where the host provides the session # 1. Scratch directory, then connect (inside SPCS; elsewhere use the §3 order). No credentials in the notebook. kumo.configure(scratch_dir=Path('/tmp/kumo-run') / 'scratch') @@ -856,10 +868,12 @@ targets a superseded build and needs rewriting against §10 — do not copy its - [ ] No code cell writes anywhere unless a destination was confirmed; an example of how to add one later is markdown text, not a code cell. - [ ] Smoke-test sizing called out as such. -- [ ] `write_mode` and any population replacement as Node 1d confirmed them (§9, §7.1). +- [ ] `write_mode` and any population replacement as Node 1d confirmed them (§9, §7.1); the + staging cell stops when the population name differs from `POPULATION_CONFIRMED`. - [ ] Destination in its connector's form (§9.2), reported with its exact name — no `_predictions` suffix. -- [ ] Written destination read back and row count compared to `prediction_result.num_rows`. +- [ ] Written destination read back and row count compared to `prediction_result.num_rows` — + for APPEND, the count added since before `predict` (§9.1). - [ ] Success judged by blocking calls returning without raising — not by a status value. --- From 0feaf8828a76455cf8c577781e97d057911efa9b Mon Sep 17 00:00:00 2001 From: igutierrez Date: Thu, 12 Nov 2026 10:37:39 -0600 Subject: [PATCH 15/35] kumo-data-science: restate the binary threshold for local predictions too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node 1d showed the threshold only when predictions are written, yet the notebook always calls predict. Without a threshold (and none recorded with the model), Trainer.predict returns True_PROB and leaves TARGET_PRED null (kumo.engine.trainer.trainer _predict); it has no 0.5 default of its own. The gate, intake_contract.md and kumo_finetuning_sdk.md §9/§12 now say so and pass the confirmed threshold whether predictions are written or local; the worked example shows the 0.5 [default] line. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- agent-skills/skills/kumo-data-science/SKILL.md | 2 +- .../skills/kumo-data-science/accept/intake_contract.md | 2 +- .../kumo-data-science/validate/examples/good_request.md | 1 + .../kumoai-reference/references/kumo_finetuning_sdk.md | 7 ++++--- skills/kumo-data-science/SKILL.md | 4 ++-- skills/kumo-data-science/accept/intake_contract.md | 2 +- .../kumoai-reference/references/kumo_finetuning_sdk.md | 7 ++++--- skills/kumo-data-science/validate/examples/good_request.md | 1 + 8 files changed, 15 insertions(+), 11 deletions(-) diff --git a/agent-skills/skills/kumo-data-science/SKILL.md b/agent-skills/skills/kumo-data-science/SKILL.md index fdc1ea534..5b52ef8aa 100644 --- a/agent-skills/skills/kumo-data-science/SKILL.md +++ b/agent-skills/skills/kumo-data-science/SKILL.md @@ -232,7 +232,7 @@ generated before the task is confirmed: - Task type: - Output: predictions written to in the same notebook | notebook only, no predictions written - Scoring population (static tasks): rows whose is NULL | all rows, staged at -- Threshold (binary classification, predictions written): +- Threshold (binary classification, written or local): - Embeddings: ``` diff --git a/agent-skills/skills/kumo-data-science/accept/intake_contract.md b/agent-skills/skills/kumo-data-science/accept/intake_contract.md index a2ffe2f7e..cafae4f8f 100644 --- a/agent-skills/skills/kumo-data-science/accept/intake_contract.md +++ b/agent-skills/skills/kumo-data-science/accept/intake_contract.md @@ -99,7 +99,7 @@ confirmed **before** any PQL is written. | **`output_location`** | none — training yields a notebook; when given, the same notebook also predicts into it (SKILL.md Step 2) | | **artifact format** | one notebook (user may ask for a script) | | **embeddings** | not included; add only when the user explicitly asks (see below) | -| **binary classification threshold** | `0.5`, shown `[default]` at Node 1d whenever a binary classification writes predictions. `predict` requires one and has no default of its own (`kumoai-reference/references/kumo_finetuning_sdk.md` §9) | +| **binary classification threshold** | `0.5`, shown `[default]` at Node 1d for every binary classification, whether predictions are written or stay local. Without one, `predict` leaves `TARGET_PRED` null; it has no default of its own (`kumoai-reference/references/kumo_finetuning_sdk.md` §9) | | **metrics** | by task type, see `SKILL.md` | If the user supplies an `output_location`, carry it verbatim into the task interpretation gate diff --git a/agent-skills/skills/kumo-data-science/validate/examples/good_request.md b/agent-skills/skills/kumo-data-science/validate/examples/good_request.md index 86d53db4a..36455ad61 100644 --- a/agent-skills/skills/kumo-data-science/validate/examples/good_request.md +++ b/agent-skills/skills/kumo-data-science/validate/examples/good_request.md @@ -57,6 +57,7 @@ Agent replies — then stops: > - Window: 0 to 30 days after the prediction anchor > - Task type: temporal binary classification > - Output: notebook only, no predictions written +> - Threshold: 0.5 [default] > - Embeddings: not requested > > Shall I generate the notebook? diff --git a/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md b/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md index c5aaf7419..26566961c 100644 --- a/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md +++ b/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md @@ -604,8 +604,9 @@ PredictionResult.export(output_config) (`kumo_core.prediction_options.MAX_LINKS_TO_PREDICT`); for more than 500 per entity, export embeddings instead. Multiclass and multilabel accept it optionally and fall back to the query's `RANK TOP K` when it has one. Regression and binary classification reject it. -- Binary classification requires an explicit `binary_classification_threshold`; there is no SDK - default. Use the one confirmed at Node 1d (§12). +- Binary classification: without `binary_classification_threshold` (and none recorded with the + model), `predict` returns `True_PROB` and leaves `TARGET_PRED` null; it has no 0.5 default of + its own. Always pass the one confirmed at Node 1d (§12), written or local alike. - `binary_classification_threshold` and `num_classes_to_return` are arguments of `predict`, and only of `predict`. `PredictionOutputConfig` describes where to write, not how to predict: it takes neither, and passing one is a `TypeError`. Whatever the destination, the config carries @@ -828,7 +829,7 @@ targets a superseded build and needs rewriting against §10 — do not copy its | PQL `query=` | Generated per `kumoai-reference/references/pql_writing.md` | | `prediction_time=` | A known in-range **naive** `datetime`; **cannot be computed** | | `predict(num_classes_to_return=N)` | Results per entity the user asked for; defaults to `RANK TOP K` but may differ. Link prediction writing predictions requires it, 1–500; no cap otherwise (§9) | -| `predict(binary_classification_threshold=)` | Required for binary classification; never on `PredictionOutputConfig`. The threshold restated at Node 1d: the user's, else `0.5 [default]` (`kumo-data-science/accept/intake_contract.md`) | +| `predict(binary_classification_threshold=)` | Always passed for binary classification, written or local; never on `PredictionOutputConfig`. The threshold restated at Node 1d: the user's, else `0.5 [default]` (`kumo-data-science/accept/intake_contract.md`) | | Population table (static task, rows that already have a target) | Confirmed at Node 1d (§7.1) | | `PredictionOutputConfig(...)` + output connector | Output destination in its connector's form (§9.2); report it back unchanged — `DB.SCHEMA.TABLE`, or the path | diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index 8494a6a48..9f9b41706 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-12-2507a7fc-dirty`** +**`SKILL-BUILD: 2026-11-12-21c1f7ab-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. @@ -232,7 +232,7 @@ generated before the task is confirmed: - Task type: - Output: predictions written to in the same notebook | notebook only, no predictions written - Scoring population (static tasks): rows whose is NULL | all rows, staged at -- Threshold (binary classification, predictions written): +- Threshold (binary classification, written or local): - Embeddings: ``` diff --git a/skills/kumo-data-science/accept/intake_contract.md b/skills/kumo-data-science/accept/intake_contract.md index a2ffe2f7e..cafae4f8f 100644 --- a/skills/kumo-data-science/accept/intake_contract.md +++ b/skills/kumo-data-science/accept/intake_contract.md @@ -99,7 +99,7 @@ confirmed **before** any PQL is written. | **`output_location`** | none — training yields a notebook; when given, the same notebook also predicts into it (SKILL.md Step 2) | | **artifact format** | one notebook (user may ask for a script) | | **embeddings** | not included; add only when the user explicitly asks (see below) | -| **binary classification threshold** | `0.5`, shown `[default]` at Node 1d whenever a binary classification writes predictions. `predict` requires one and has no default of its own (`kumoai-reference/references/kumo_finetuning_sdk.md` §9) | +| **binary classification threshold** | `0.5`, shown `[default]` at Node 1d for every binary classification, whether predictions are written or stay local. Without one, `predict` leaves `TARGET_PRED` null; it has no default of its own (`kumoai-reference/references/kumo_finetuning_sdk.md` §9) | | **metrics** | by task type, see `SKILL.md` | If the user supplies an `output_location`, carry it verbatim into the task interpretation gate diff --git a/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md b/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md index c5aaf7419..26566961c 100644 --- a/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md +++ b/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md @@ -604,8 +604,9 @@ PredictionResult.export(output_config) (`kumo_core.prediction_options.MAX_LINKS_TO_PREDICT`); for more than 500 per entity, export embeddings instead. Multiclass and multilabel accept it optionally and fall back to the query's `RANK TOP K` when it has one. Regression and binary classification reject it. -- Binary classification requires an explicit `binary_classification_threshold`; there is no SDK - default. Use the one confirmed at Node 1d (§12). +- Binary classification: without `binary_classification_threshold` (and none recorded with the + model), `predict` returns `True_PROB` and leaves `TARGET_PRED` null; it has no 0.5 default of + its own. Always pass the one confirmed at Node 1d (§12), written or local alike. - `binary_classification_threshold` and `num_classes_to_return` are arguments of `predict`, and only of `predict`. `PredictionOutputConfig` describes where to write, not how to predict: it takes neither, and passing one is a `TypeError`. Whatever the destination, the config carries @@ -828,7 +829,7 @@ targets a superseded build and needs rewriting against §10 — do not copy its | PQL `query=` | Generated per `kumoai-reference/references/pql_writing.md` | | `prediction_time=` | A known in-range **naive** `datetime`; **cannot be computed** | | `predict(num_classes_to_return=N)` | Results per entity the user asked for; defaults to `RANK TOP K` but may differ. Link prediction writing predictions requires it, 1–500; no cap otherwise (§9) | -| `predict(binary_classification_threshold=)` | Required for binary classification; never on `PredictionOutputConfig`. The threshold restated at Node 1d: the user's, else `0.5 [default]` (`kumo-data-science/accept/intake_contract.md`) | +| `predict(binary_classification_threshold=)` | Always passed for binary classification, written or local; never on `PredictionOutputConfig`. The threshold restated at Node 1d: the user's, else `0.5 [default]` (`kumo-data-science/accept/intake_contract.md`) | | Population table (static task, rows that already have a target) | Confirmed at Node 1d (§7.1) | | `PredictionOutputConfig(...)` + output connector | Output destination in its connector's form (§9.2); report it back unchanged — `DB.SCHEMA.TABLE`, or the path | diff --git a/skills/kumo-data-science/validate/examples/good_request.md b/skills/kumo-data-science/validate/examples/good_request.md index 86d53db4a..36455ad61 100644 --- a/skills/kumo-data-science/validate/examples/good_request.md +++ b/skills/kumo-data-science/validate/examples/good_request.md @@ -57,6 +57,7 @@ Agent replies — then stops: > - Window: 0 to 30 days after the prediction anchor > - Task type: temporal binary classification > - Output: notebook only, no predictions written +> - Threshold: 0.5 [default] > - Embeddings: not requested > > Shall I generate the notebook? From 465899b2d301ef0713f8bb5407d8bb0c28afa480 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Thu, 12 Nov 2026 13:35:10 -0600 Subject: [PATCH 16/35] publish_nvskills: publish each nested skill as its own catalog skill NVIDIA/skills' sync deletes every SKILL.md directory that lacks its own skill.oms.sig, skill-card.md and eval dataset. The generic package has eight: the router and seven nested skills, which stay nested so the router's links resolve. Treat each as a catalog skill: - catalog files live under agent-skills/nvskills/kumo-data-science/ in the package layout, and each must be a skill's skill-card.md or evals/ file; - once that directory exists, the build fails unless every skill directory has a card and an evals/*.json, so a new nested skill cannot be dropped silently; until then publish_nvskills.py lists the missing inputs; - BENCHMARK.md and skill.oms.sig are kept at any depth, not only at the root. The README checklist now lists the card and evals for all eight skills. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- agent-skills/README.md | 47 +++++++++--- agent-skills/scripts/publish_nvskills.py | 76 ++++++++++++++++--- agent-skills/tests/test_publish_nvskills.py | 81 +++++++++++++++++---- 3 files changed, 172 insertions(+), 32 deletions(-) diff --git a/agent-skills/README.md b/agent-skills/README.md index 50ca03d70..3b832c333 100644 --- a/agent-skills/README.md +++ b/agent-skills/README.md @@ -133,11 +133,21 @@ once an hour. It accepts only content that NVSkills CI has evaluated and signed. only on PRs that change a root-level `skills/` directory. So the catalog reads a built copy, `skills/kumo-data-science/` at the repository root, not `agent-skills/skills/`: -| Path | Written by | +The catalog treats every directory with a `SKILL.md` as a separate skill, and its sync deletes any +such directory that lacks its own `skill.oms.sig`, `skill-card.md` and eval dataset. The package has +eight: the router `skills/kumo-data-science/` and the seven skills nested in it +(`kumoai-reference/` and the six `kumo-*/` capability skills). They stay nested so the router's +relative links resolve, and each of the eight is its own catalog skill: + +| Path, in each of the eight skill directories | Written by | |---|---| -| `skills/kumo-data-science/` (everything else) | `agent-skills/scripts/publish_nvskills.py`: the `generic` package | -| `skills/kumo-data-science/skill-card.md`, `evals/` | copied from `agent-skills/nvskills/kumo-data-science/` | -| `skills/kumo-data-science/BENCHMARK.md`, `skill.oms.sig` | NVSkills CI, in your PR; never by hand | +| everything else | `agent-skills/scripts/publish_nvskills.py`: the `generic` package | +| `skill-card.md`, `evals/` | copied from the same path under `agent-skills/nvskills/kumo-data-science/`, e.g. `agent-skills/nvskills/kumo-data-science/kumo-model-tuning/skill-card.md` | +| `BENCHMARK.md`, `skill.oms.sig` | NVSkills CI, in your PR; never by hand | + +Once `agent-skills/nvskills/` exists, the build fails unless every skill has both a card and an +`evals/*.json`, so a new nested skill cannot be published without them. Until then +`publish_nvskills.py` lists what is missing. Whatever is on `main` is what the catalog publishes. To change the skill: @@ -157,13 +167,30 @@ merge. Run `/nvskills-ci` on reverts too. ### Before the first publish -None of this exists yet; the catalog publishes nothing until all of it does: +None of this exists yet; the catalog publishes nothing until all of it does. The card follows the +template in NVIDIA/skills `docs/skill-cards.mdx`; the evals follow the `evals/evals.json` of a +published catalog skill. Each skill needs its own, under +`agent-skills/nvskills/kumo-data-science/`: -- [ ] `agent-skills/nvskills/kumo-data-science/skill-card.md` -- [ ] `agent-skills/nvskills/kumo-data-science/evals/evals.json` +| Skill | Card | Evals | +|---|---|---| +| `kumo-data-science` (router) | [ ] `skill-card.md` | [ ] `evals/evals.json` | +| `kumoai-reference` | [ ] `kumoai-reference/skill-card.md` | [ ] `kumoai-reference/evals/evals.json` | +| `kumo-custom-populations` | [ ] `kumo-custom-populations/skill-card.md` | [ ] `kumo-custom-populations/evals/evals.json` | +| `kumo-graph-visualization` | [ ] `kumo-graph-visualization/skill-card.md` | [ ] `kumo-graph-visualization/evals/evals.json` | +| `kumo-model-explanations` | [ ] `kumo-model-explanations/skill-card.md` | [ ] `kumo-model-explanations/evals/evals.json` | +| `kumo-model-persistence` | [ ] `kumo-model-persistence/skill-card.md` | [ ] `kumo-model-persistence/evals/evals.json` | +| `kumo-model-tuning` | [ ] `kumo-model-tuning/skill-card.md` | [ ] `kumo-model-tuning/evals/evals.json` | +| `kumo-training-diagnostics` | [ ] `kumo-training-diagnostics/skill-card.md` | [ ] `kumo-training-diagnostics/evals/evals.json` | + +- [ ] confirmation from the NVSkills CI maintainers that `/nvskills-ci` evaluates and signs each + of the eight directories, writing `BENCHMARK.md` and `skill.oms.sig` into each. No skill in + the catalog today nests another, so this layout has not been through NVSkills CI before - [ ] the `NVSKILLS_CI_DISPATCH_TOKEN` repository secret, from the NVSkills CI maintainers; `request-nvskills-ci.yml` fails without it -- [ ] a signed copy on `main`: a PR through steps 1–4 whose `BENCHMARK.md` says PASS +- [ ] a signed copy on `main`: a PR through steps 1–4 whose `BENCHMARK.md` says PASS in every + skill directory - [ ] the onboarding PR to NVIDIA/skills adding `components.d/kumo-engine.yml`, pointing at - `skills/kumo-data-science`. Its `verify-source-onboarding` check passes only once the - signed copy is on `main`. + `skills/kumo-data-science`. Its `verify-source-onboarding` check verifies only the paths the + file lists, so list all eight to have each checked. It passes only once the signed copy is + on `main`. diff --git a/agent-skills/scripts/publish_nvskills.py b/agent-skills/scripts/publish_nvskills.py index 58b5b6680..76e2e1da6 100644 --- a/agent-skills/scripts/publish_nvskills.py +++ b/agent-skills/scripts/publish_nvskills.py @@ -17,17 +17,24 @@ the skill runs it and commits the result, and `tests/test_publish_nvskills.py` fails the PR when it did not. NVSkills CI signs that same PR, so merging it to `main` is what publishes. +The catalog treats every directory holding a `SKILL.md` as its own skill, and its sync deletes +any such directory that lacks its own `skill.oms.sig`, `skill-card.md` and eval dataset. The +package has eight: the router and the seven skills nested inside it, which stay nested so the +router's relative links resolve. Each of the eight is a catalog skill here. + Three kinds of file sit in the published directory: * the package, rebuilt from `agent-skills/skills/` on every run; -* the catalog files that NVSkills requires and that we write: `skill-card.md` and - `evals/*.json`. Their source is `agent-skills/nvskills/kumo-data-science/`, and they are - copied in on every run. A catalog file may not replace a package file; +* the catalog files that NVSkills requires and that we write: each skill's `skill-card.md` and + `evals/*.json`. Their source is `agent-skills/nvskills/kumo-data-science/`, laid out like the + package (a nested skill's files under `/`), and they are copied in on every run. A + catalog file must belong to a skill directory and may not replace a package file. Once that + source exists, the build fails unless every skill has both, so the sync never drops one; * `BENCHMARK.md` and `skill.oms.sig`, which NVSkills CI writes into the pull request. They are never edited here, and the build fails if the package or the catalog files carry either - name at any depth. A refresh leaves them in place, so they describe the previous content until - NVSkills CI runs again. The catalog then keeps serving the last signed version rather than - dropping the skill. + name at any depth. A refresh leaves them in place at any depth, so they describe the previous + content until NVSkills CI runs again. The catalog then keeps serving the last signed version + rather than dropping the skill. `--check` ignores the `SKILL-BUILD` stamp, which names the commit that built the package and so differs on every commit even when no skill file changed. @@ -54,6 +61,37 @@ CATALOG_FILES = package_skill.ROOT / 'nvskills' / package_skill.ROUTER # Written by NVSkills CI into the pull request; never generated or deleted here. SIGNED = frozenset({'BENCHMARK.md', 'skill.oms.sig'}) +# What each skill directory needs from the catalog files; the sync drops a skill without them. +CARD = 'skill-card.md' +EVALS = 'evals' + + +def skill_dirs(pkg: Path) -> list[str]: + """Each directory of `pkg` that the catalog treats as a skill, `.` for the router.""" + return sorted(p.parent.relative_to(pkg).as_posix() for p in pkg.rglob('SKILL.md')) + + +def owner(rel: Path) -> str | None: + """The skill directory a catalog file belongs to, or None if it is not a catalog input.""" + parts = rel.parts + if parts[-1] == CARD: + return Path(*parts[:-1]).as_posix() if len(parts) > 1 else '.' + if EVALS in parts[:-1]: + i = parts.index(EVALS) + return Path(*parts[:i]).as_posix() if i else '.' + return None + + +def missing(pkg: Path) -> list[str]: + """The catalog inputs a skill directory of `pkg` lacks, as package-relative paths.""" + out = [] + for skill in skill_dirs(pkg): + d = pkg / skill + if not (d / CARD).is_file(): + out.append((Path(skill) / CARD).as_posix()) + if not any((d / EVALS).glob('*.json')): + out.append((Path(skill) / EVALS / '*.json').as_posix()) + return out def build(out: Path, catalog_files: Path = CATALOG_FILES) -> Path: @@ -67,19 +105,36 @@ def build(out: Path, catalog_files: Path = CATALOG_FILES) -> Path: if signed: raise SystemExit(f'{HOST} package ships {signed}: written by NVSkills CI, not by hand') if catalog_files.is_dir(): + skills = skill_dirs(pkg) for path in sorted(catalog_files.rglob('*')): if path.is_file(): rel = path.relative_to(catalog_files) if path.name in SIGNED: raise SystemExit(f'{path}: {rel} is written by NVSkills CI, not by hand') + if owner(rel) not in skills: + raise SystemExit(f'{path}: {rel} is not the {CARD} or {EVALS}/ of a skill ' + f'in the {HOST} package ({skills})') target = pkg / rel if target.exists(): raise SystemExit(f'{path}: {rel} is already in the {HOST} package') target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(path, target) + # The catalog sync deletes a skill directory without its own card and evals, and with + # it the files the router links to. Before the first one is written nothing is + # published, so the gap is reported (`not_ready`) rather than fatal. + gaps = missing(pkg) + if gaps: + raise SystemExit(f'{catalog_files}: the catalog would drop these skills; add ' + f'{gaps}') return pkg +def not_ready(catalog_files: Path = CATALOG_FILES) -> list[str]: + """Catalog inputs still to write before the first publish; empty once they all exist.""" + with tempfile.TemporaryDirectory() as tmp: + return missing(build(Path(tmp), catalog_files)) + + def files(root: Path) -> dict[str, bytes]: """Map each file under `root` that we own to its bytes, with the build stamp normalized.""" if not root.is_dir(): @@ -87,7 +142,7 @@ def files(root: Path) -> dict[str, bytes]: out = {} for path in sorted(root.rglob('*')): rel = path.relative_to(root).as_posix() - if path.is_file() and rel not in SIGNED: + if path.is_file() and path.name not in SIGNED: data = path.read_bytes() if rel == 'SKILL.md': text = package_skill.MARKER.sub('SKILL-BUILD: x', data.decode('utf-8')) @@ -115,8 +170,7 @@ def publish(dest: Path = PUBLISHED, catalog_files: Path = CATALOG_FILES) -> list pkg = build(Path(tmp), catalog_files) if dest.is_dir(): for path in sorted(dest.rglob('*'), reverse=True): - rel = path.relative_to(dest).as_posix() - if path.is_file() and rel not in SIGNED: + if path.is_file() and path.name not in SIGNED: path.unlink() elif path.is_dir() and not any(path.iterdir()): path.rmdir() @@ -144,6 +198,10 @@ def main() -> None: raise SystemExit(1) else: print(f'{where}/ refreshed: {len(changes)} file(s)') + todo = not_ready() + if todo: + print(f'not publishable yet: the catalog would drop every skill until ' + f'{CATALOG_FILES.relative_to(REPO)}/ supplies {todo}', file=sys.stderr) if __name__ == '__main__': diff --git a/agent-skills/tests/test_publish_nvskills.py b/agent-skills/tests/test_publish_nvskills.py index 2eccac9ec..25496335e 100644 --- a/agent-skills/tests/test_publish_nvskills.py +++ b/agent-skills/tests/test_publish_nvskills.py @@ -15,12 +15,17 @@ spec.loader.exec_module(publish_nvskills) +# The router and every skill the generic package nests inside it. +SKILLS = ['.', 'kumoai-reference', *publish_nvskills.package_skill.CAPABILITY] + + @pytest.fixture def catalog(tmp_path: Path) -> Path: root = tmp_path / 'catalog' - (root / 'evals').mkdir(parents=True) - (root / 'evals' / 'evals.json').write_text('{"skill_name": "kumo-data-science"}\n') - (root / 'skill-card.md').write_text('## Description\n') + for skill in SKILLS: + (root / skill / 'evals').mkdir(parents=True, exist_ok=True) + (root / skill / 'evals' / 'evals.json').write_text(f'{{"skill_name": "{skill}"}}\n') + (root / skill / 'skill-card.md').write_text('## Description\n') return root @@ -31,25 +36,65 @@ def test_publish_writes_the_generic_package_and_catalog_files(tmp_path: Path, assert (dest / 'SKILL.md').is_file() assert (dest / 'hosts' / 'generic' / 'adapter.md').is_file() assert not (dest / 'hosts' / 'snowflake-coco').exists() - assert (dest / 'evals' / 'evals.json').is_file() - assert (dest / 'skill-card.md').is_file() + assert sorted(publish_nvskills.skill_dirs(dest)) == sorted(SKILLS) + for skill in SKILLS: + assert (dest / skill / 'evals' / 'evals.json').is_file() + assert (dest / skill / 'skill-card.md').is_file() + assert publish_nvskills.missing(dest) == [] assert publish_nvskills.stale(dest, catalog) == [] +@pytest.mark.parametrize('rel', ['kumoai-reference/skill-card.md', + 'kumo-model-tuning/evals/evals.json', 'evals/evals.json']) +def test_every_skill_directory_needs_its_own_card_and_evals(tmp_path: Path, catalog: Path, + rel: str) -> None: + """The catalog sync deletes a SKILL.md directory without its own card and evals. + + For a nested skill that also breaks the router's links into it, so the build refuses. + """ + (catalog / rel).unlink() + dest = tmp_path / 'kumo-data-science' + with pytest.raises(SystemExit, match='the catalog would drop these skills'): + publish_nvskills.publish(dest, catalog) + assert not dest.exists() + + +def test_without_catalog_files_every_input_is_reported_missing(tmp_path: Path) -> None: + absent = tmp_path / 'no-catalog' + dest = tmp_path / 'kumo-data-science' + assert publish_nvskills.publish(dest, absent) + todo = publish_nvskills.not_ready(absent) + for skill in SKILLS: + assert (Path(skill) / 'skill-card.md').as_posix() in todo + assert (Path(skill) / 'evals' / '*.json').as_posix() in todo + assert len(todo) == 2 * len(SKILLS) + + +@pytest.mark.parametrize('rel', ['notes.md', 'SKILL.md', 'hosts/generic/adapter.md', + 'hosts/skill-card.md', 'hosts/evals/evals.json', + 'kumo-model-tunning/skill-card.md']) +def test_catalog_files_must_belong_to_a_skill(tmp_path: Path, catalog: Path, rel: str) -> None: + (catalog / rel).parent.mkdir(parents=True, exist_ok=True) + (catalog / rel).write_text('stray\n') + with pytest.raises(SystemExit, match='is not the skill-card.md or evals/ of a skill'): + publish_nvskills.publish(tmp_path / 'kumo-data-science', catalog) + + def test_publish_keeps_signed_files_and_drops_files_no_longer_shipped(tmp_path: Path, catalog: Path) -> None: dest = tmp_path / 'kumo-data-science' publish_nvskills.publish(dest, catalog) - for name in publish_nvskills.SIGNED: - (dest / name).write_text('from NVSkills CI\n') + signed = [dest / skill / name for skill in SKILLS for name in publish_nvskills.SIGNED] + for path in signed: + path.write_text('from NVSkills CI\n') (dest / 'old' / 'removed.md').parent.mkdir() (dest / 'old' / 'removed.md').write_text('gone\n') assert publish_nvskills.stale(dest, catalog) == ['- old/removed.md'] publish_nvskills.publish(dest, catalog) assert not (dest / 'old').exists() - for name in publish_nvskills.SIGNED: - assert (dest / name).read_text() == 'from NVSkills CI\n' + for path in signed: + assert path.read_text() == 'from NVSkills CI\n' def test_check_ignores_the_build_stamp_but_not_content(tmp_path: Path, catalog: Path) -> None: @@ -65,7 +110,9 @@ def test_check_ignores_the_build_stamp_but_not_content(tmp_path: Path, catalog: assert publish_nvskills.stale(dest, catalog) == ['~ SKILL.md'] -@pytest.mark.parametrize('rel', ['skill.oms.sig', 'evals/skill.oms.sig', 'evals/BENCHMARK.md']) +@pytest.mark.parametrize('rel', ['skill.oms.sig', 'evals/skill.oms.sig', 'evals/BENCHMARK.md', + 'kumoai-reference/skill.oms.sig', + 'kumo-model-tuning/BENCHMARK.md']) def test_catalog_files_cannot_supply_signed_files(tmp_path: Path, catalog: Path, rel: str) -> None: (catalog / rel).write_text('forged\n') @@ -73,11 +120,19 @@ def test_catalog_files_cannot_supply_signed_files(tmp_path: Path, catalog: Path, publish_nvskills.publish(tmp_path / 'kumo-data-science', catalog) -@pytest.mark.parametrize('rel', ['SKILL.md', 'hosts/generic/adapter.md']) +@pytest.mark.parametrize('rel', ['skill-card.md', 'kumo-model-tuning/evals/evals.json']) def test_catalog_files_cannot_replace_package_files(tmp_path: Path, catalog: Path, + monkeypatch: pytest.MonkeyPatch, rel: str) -> None: - (catalog / rel).parent.mkdir(parents=True, exist_ok=True) - (catalog / rel).write_text('replaced\n') + build = publish_nvskills.package_skill.build + + def shipping(host: str, dist: Path) -> Path: + pkg = build(host, dist) + (pkg / rel).parent.mkdir(parents=True, exist_ok=True) + (pkg / rel).write_text('from the package\n') + return pkg + + monkeypatch.setattr(publish_nvskills.package_skill, 'build', shipping) with pytest.raises(SystemExit, match='already in the generic package'): publish_nvskills.publish(tmp_path / 'kumo-data-science', catalog) From 2105a3a21ee3741b7ed0ea421e3dc3090d96a08b Mon Sep 17 00:00:00 2001 From: igutierrez Date: Thu, 12 Nov 2026 13:35:52 -0600 Subject: [PATCH 17/35] kumo-graph-visualization: rerunning a script redraws its own diagram Reverts the part of d9d0ab883 that wrote a new random-named SVG when .graph.svg existed. That name derives from the script's own name, so the only file it replaces is the diagram an earlier run of the same script wrote, and a rerun should redraw it; the random names only accumulated stale diagrams. The skill now says why the overwrite is intended. Kept from d9d0ab883: the Markdown schema goes to a fresh tempfile.mkdtemp() directory and its path is printed, and an export without a user-named path gets one that does not exist yet while an existing user-named file is confirmed before it is replaced. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- .../skills/kumo-graph-visualization/SKILL.md | 14 +++++--------- skills/kumo-data-science/SKILL.md | 2 +- .../kumo-graph-visualization/SKILL.md | 14 +++++--------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/agent-skills/skills/kumo-graph-visualization/SKILL.md b/agent-skills/skills/kumo-graph-visualization/SKILL.md index 422f2722a..6c29d3ac9 100644 --- a/agent-skills/skills/kumo-graph-visualization/SKILL.md +++ b/agent-skills/skills/kumo-graph-visualization/SKILL.md @@ -33,18 +33,14 @@ Continue the workflow normally; pause for visual review only if the user asks. Save the diagram next to the artifact, named after it (`churn_30d_train.graph.svg` for `churn_30d_train.py`), before training, and report its path. `visualize` overwrites an existing -file, so when that name is taken — by an earlier run or anything else — write a new file beside -it instead: +file, and here that is intended: the name derives from the script's own name, so the only file it +can replace is the diagram an earlier run of the same script wrote, and a rerun should redraw it +for the graph it trains on: ```python -import tempfile from pathlib import Path -graph_path = Path(__file__).with_suffix('.graph.svg').resolve() -if graph_path.exists(): # never replace a file: take a new name beside it - with tempfile.NamedTemporaryFile(dir=graph_path.parent, prefix=graph_path.stem + '-', - suffix='.svg', delete=False) as reserved: - graph_path = Path(reserved.name) +graph_path = Path(__file__).with_suffix('.graph.svg').resolve() # this script's own diagram if not hasattr(graph, 'visualize'): print('Graph preview needs a kumo-engine build including #134; upgrade when available. Continuing.') else: @@ -62,7 +58,7 @@ The viewer supports up to 30 tables and does not support self-relations. Display the graph in a notebook or save the format the user requests: `.svg` for a diagram, `.pdf` for a diagram and complete schema index, or `.md` for a Mermaid diagram and schema tables. `graph.visualize(EXPORT_PATH, show_cols=True)` also shows ordinary columns in the diagram. -`EXPORT_PATH` is the path the user named. When they named none, choose one that does not exist yet, as for scripts, and report it; when the named file exists, say so and confirm before replacing it. +`EXPORT_PATH` is the path the user named. When they named none, choose one that does not exist yet and report it; when the named file exists, say so and confirm before replacing it. Use `layout='standard'` or `layout='compact'` to select an arrangement; the default is `'auto'`. Export paths belong to the Python runtime; use `hosts//runtime.md` for display and file retrieval. diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index 9f9b41706..0989d9af6 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-12-21c1f7ab-dirty`** +**`SKILL-BUILD: 2026-11-12-465899b2-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/kumo-graph-visualization/SKILL.md b/skills/kumo-data-science/kumo-graph-visualization/SKILL.md index 422f2722a..6c29d3ac9 100644 --- a/skills/kumo-data-science/kumo-graph-visualization/SKILL.md +++ b/skills/kumo-data-science/kumo-graph-visualization/SKILL.md @@ -33,18 +33,14 @@ Continue the workflow normally; pause for visual review only if the user asks. Save the diagram next to the artifact, named after it (`churn_30d_train.graph.svg` for `churn_30d_train.py`), before training, and report its path. `visualize` overwrites an existing -file, so when that name is taken — by an earlier run or anything else — write a new file beside -it instead: +file, and here that is intended: the name derives from the script's own name, so the only file it +can replace is the diagram an earlier run of the same script wrote, and a rerun should redraw it +for the graph it trains on: ```python -import tempfile from pathlib import Path -graph_path = Path(__file__).with_suffix('.graph.svg').resolve() -if graph_path.exists(): # never replace a file: take a new name beside it - with tempfile.NamedTemporaryFile(dir=graph_path.parent, prefix=graph_path.stem + '-', - suffix='.svg', delete=False) as reserved: - graph_path = Path(reserved.name) +graph_path = Path(__file__).with_suffix('.graph.svg').resolve() # this script's own diagram if not hasattr(graph, 'visualize'): print('Graph preview needs a kumo-engine build including #134; upgrade when available. Continuing.') else: @@ -62,7 +58,7 @@ The viewer supports up to 30 tables and does not support self-relations. Display the graph in a notebook or save the format the user requests: `.svg` for a diagram, `.pdf` for a diagram and complete schema index, or `.md` for a Mermaid diagram and schema tables. `graph.visualize(EXPORT_PATH, show_cols=True)` also shows ordinary columns in the diagram. -`EXPORT_PATH` is the path the user named. When they named none, choose one that does not exist yet, as for scripts, and report it; when the named file exists, say so and confirm before replacing it. +`EXPORT_PATH` is the path the user named. When they named none, choose one that does not exist yet and report it; when the named file exists, say so and confirm before replacing it. Use `layout='standard'` or `layout='compact'` to select an arrangement; the default is `'auto'`. Export paths belong to the Python runtime; use `hosts//runtime.md` for display and file retrieval. From ecf2318d8f994b5ed515ba4d700713acc59f576e Mon Sep 17 00:00:00 2001 From: igutierrez Date: Fri, 13 Nov 2026 16:30:48 -0600 Subject: [PATCH 18/35] agent-skills: address CodeRabbit's 2026-11-12 follow-ups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - hosts/generic/runtime.md §3.1: the S3 population connector takes the source's explicit identity (None under ambient credentials), so a population Arrow staged with explicit keys is also readable by Kumo. - SKILL.md Node 1d: the default population location applies only when the destination is outside the source schema and source path; the rule now says why it must stay out of both. - kumo_finetuning_sdk.md §3.2: a rootless S3Connector has no table_names(); root it at the confirmed prefix. - kumo_finetuning_sdk.md §7.1: escape quotes in Snowflake identifiers, as the Databricks recipe already does. Published copy rebuilt with publish_nvskills.py; --check passes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- agent-skills/skills/hosts/generic/runtime.md | 7 ++++--- agent-skills/skills/kumo-data-science/SKILL.md | 5 +++-- .../kumoai-reference/references/kumo_finetuning_sdk.md | 8 ++++++-- skills/kumo-data-science/SKILL.md | 7 ++++--- skills/kumo-data-science/hosts/generic/runtime.md | 7 ++++--- .../kumoai-reference/references/kumo_finetuning_sdk.md | 8 ++++++-- 6 files changed, 27 insertions(+), 15 deletions(-) diff --git a/agent-skills/skills/hosts/generic/runtime.md b/agent-skills/skills/hosts/generic/runtime.md index 75b2f4e4d..bb501fbd5 100644 --- a/agent-skills/skills/hosts/generic/runtime.md +++ b/agent-skills/skills/hosts/generic/runtime.md @@ -130,8 +130,8 @@ written = ds.dataset(OUTPUT_LOCATION.removeprefix('s3://'), format=OUTPUT_FORMAT filesystem=s3).count_rows() ``` -Without an explicit identity, emit `s3 = None`: §3.1's population staging then lets Arrow resolve -the ambient credentials itself. +Without an explicit identity, emit `identity = None` and `s3 = None`: §3.1's population staging +then lets Kumo and Arrow resolve the ambient credentials themselves. The Snowflake factory `from_spcs_env` and Snowflake's notebook secret store belong to Snowflake's own runtime. Emit them only when the user says the code runs there. @@ -289,7 +289,8 @@ system; the destination may be in another system (`adapter.md` §4) and plays no schema=schema)) pop_fs.move(final + '.partial', final) # replace only once every batch is in if POPULATION_PATH.startswith('s3://'): # the source's kind, in the §2 forms - population_connector = kumo.S3Connector(root_dir=POPULATION_PATH) + population_connector = kumo.S3Connector(root_dir=POPULATION_PATH, + identity=identity) # §2's; None under ambient elif POPULATION_PATH.startswith('gs://'): population_connector = kumo.GCSConnector(root_dir=POPULATION_PATH) else: diff --git a/agent-skills/skills/kumo-data-science/SKILL.md b/agent-skills/skills/kumo-data-science/SKILL.md index 5b52ef8aa..95b2bf812 100644 --- a/agent-skills/skills/kumo-data-science/SKILL.md +++ b/agent-skills/skills/kumo-data-science/SKILL.md @@ -240,12 +240,13 @@ generated before the task is confirmed: target is NULL (`kumoai-reference/references/kumo_finetuning_sdk.md` §7.1). If the user wants rows that already have a target scored (for example "every order"), the notebook stages an explicit population table. That is a write, and it goes in the source's system, so name it here: next to the confirmed -destination by default when the destination is in the source's system — the table +destination by default when the destination is in the source's system but not in the source schema +or under the source path — the table `_POPULATION`, or for a path the sibling directory `_population/` (`s3://b/scores/` → `s3://b/scores_population/`), never inside the destination, which must stay empty; otherwise, or when there is no destination, ask for a schema or path in the source's system. Never stage it in -the source schema. +the source schema or under the source path, where it would read as a source table. **Name what a run replaces.** The notebook writes predictions with `OVERWRITE` and stages a population with `CREATE OR REPLACE` (a file: overwritten), on every run. Check from the catalog diff --git a/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md b/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md index 26566961c..f1b0b943b 100644 --- a/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md +++ b/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md @@ -208,7 +208,8 @@ connector.has_table('orders') # -> bool `.head()`** — do not emit one. **Assert `table_names()` before building tables.** It is the cheap proof the connector reaches the -data, and it fails early with a clear error instead of deep inside a snapshot. +data, and it fails early with a clear error instead of deep inside a snapshot. A rootless +`S3Connector` (`root_dir=None`) has no `table_names()`, so root it at the confirmed prefix. **Use the catalog's exact spelling for table and column names.** `connector['
']` tries the exact name, then its upper-cased form, so `connector['ENTITY_PROFILE']` never finds a lower-case @@ -470,11 +471,14 @@ says how a population is staged, or that it is not covered there: `ENTITY1` in the confirmed population table and pass it explicitly: ```python +def q(name): # a quoted Snowflake identifier + return '"' + name.replace('"', '""') + '"' + if f'{POP_DATABASE}.{POP_SCHEMA}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') session.sql( # the session from §3.3 f'CREATE OR REPLACE TABLE {POP_DATABASE}.{POP_SCHEMA}.{POP_TABLE} AS ' - f'SELECT "{ENTITY_KEY}" AS ENTITY1 FROM {SOURCE_DATABASE}.{SOURCE_SCHEMA}."{ENTITY_TABLE}"' + f'SELECT {q(ENTITY_KEY)} AS ENTITY1 FROM {SOURCE_DATABASE}.{SOURCE_SCHEMA}.{q(ENTITY_TABLE)}' ).collect() population_connector = kumo.SnowflakeConnector.from_spcs_env( # or from_connection_toml, §3 warehouse=WAREHOUSE, database=POP_DATABASE, schema_name=POP_SCHEMA, diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index 0989d9af6..29216f9a1 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-12-465899b2-dirty`** +**`SKILL-BUILD: 2026-11-12-2105a3a2-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. @@ -240,12 +240,13 @@ generated before the task is confirmed: target is NULL (`kumoai-reference/references/kumo_finetuning_sdk.md` §7.1). If the user wants rows that already have a target scored (for example "every order"), the notebook stages an explicit population table. That is a write, and it goes in the source's system, so name it here: next to the confirmed -destination by default when the destination is in the source's system — the table +destination by default when the destination is in the source's system but not in the source schema +or under the source path — the table `_POPULATION`, or for a path the sibling directory `_population/` (`s3://b/scores/` → `s3://b/scores_population/`), never inside the destination, which must stay empty; otherwise, or when there is no destination, ask for a schema or path in the source's system. Never stage it in -the source schema. +the source schema or under the source path, where it would read as a source table. **Name what a run replaces.** The notebook writes predictions with `OVERWRITE` and stages a population with `CREATE OR REPLACE` (a file: overwritten), on every run. Check from the catalog diff --git a/skills/kumo-data-science/hosts/generic/runtime.md b/skills/kumo-data-science/hosts/generic/runtime.md index 75b2f4e4d..bb501fbd5 100644 --- a/skills/kumo-data-science/hosts/generic/runtime.md +++ b/skills/kumo-data-science/hosts/generic/runtime.md @@ -130,8 +130,8 @@ written = ds.dataset(OUTPUT_LOCATION.removeprefix('s3://'), format=OUTPUT_FORMAT filesystem=s3).count_rows() ``` -Without an explicit identity, emit `s3 = None`: §3.1's population staging then lets Arrow resolve -the ambient credentials itself. +Without an explicit identity, emit `identity = None` and `s3 = None`: §3.1's population staging +then lets Kumo and Arrow resolve the ambient credentials themselves. The Snowflake factory `from_spcs_env` and Snowflake's notebook secret store belong to Snowflake's own runtime. Emit them only when the user says the code runs there. @@ -289,7 +289,8 @@ system; the destination may be in another system (`adapter.md` §4) and plays no schema=schema)) pop_fs.move(final + '.partial', final) # replace only once every batch is in if POPULATION_PATH.startswith('s3://'): # the source's kind, in the §2 forms - population_connector = kumo.S3Connector(root_dir=POPULATION_PATH) + population_connector = kumo.S3Connector(root_dir=POPULATION_PATH, + identity=identity) # §2's; None under ambient elif POPULATION_PATH.startswith('gs://'): population_connector = kumo.GCSConnector(root_dir=POPULATION_PATH) else: diff --git a/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md b/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md index 26566961c..f1b0b943b 100644 --- a/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md +++ b/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md @@ -208,7 +208,8 @@ connector.has_table('orders') # -> bool `.head()`** — do not emit one. **Assert `table_names()` before building tables.** It is the cheap proof the connector reaches the -data, and it fails early with a clear error instead of deep inside a snapshot. +data, and it fails early with a clear error instead of deep inside a snapshot. A rootless +`S3Connector` (`root_dir=None`) has no `table_names()`, so root it at the confirmed prefix. **Use the catalog's exact spelling for table and column names.** `connector['
']` tries the exact name, then its upper-cased form, so `connector['ENTITY_PROFILE']` never finds a lower-case @@ -470,11 +471,14 @@ says how a population is staged, or that it is not covered there: `ENTITY1` in the confirmed population table and pass it explicitly: ```python +def q(name): # a quoted Snowflake identifier + return '"' + name.replace('"', '""') + '"' + if f'{POP_DATABASE}.{POP_SCHEMA}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') session.sql( # the session from §3.3 f'CREATE OR REPLACE TABLE {POP_DATABASE}.{POP_SCHEMA}.{POP_TABLE} AS ' - f'SELECT "{ENTITY_KEY}" AS ENTITY1 FROM {SOURCE_DATABASE}.{SOURCE_SCHEMA}."{ENTITY_TABLE}"' + f'SELECT {q(ENTITY_KEY)} AS ENTITY1 FROM {SOURCE_DATABASE}.{SOURCE_SCHEMA}.{q(ENTITY_TABLE)}' ).collect() population_connector = kumo.SnowflakeConnector.from_spcs_env( # or from_connection_toml, §3 warehouse=WAREHOUSE, database=POP_DATABASE, schema_name=POP_SCHEMA, From 2ab10b0fac6d7a67b487c6ca60b9c550d6dfe231 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Tue, 17 Nov 2026 09:47:05 -0600 Subject: [PATCH 19/35] agent-skills: parse a formatted CSV time column when staging a population The file-source staging recipe cast the entity time column with pc.cast, which only parses ISO-8601. graph_construction.md tells the agent to declare a non-ISO CSV time column with timestamp_format='%m/%d/%Y'; such a column reads as text, and the cast raised ArrowInvalid inside the write loop. Parse it with pc.strptime and the entity Table's timestamp_format (ENTITY_TIME_FORMAT) when set. Co-Authored-By: Claude Opus 5.5 --- agent-skills/skills/hosts/generic/runtime.md | 14 ++++++++++++-- skills/kumo-data-science/SKILL.md | 2 +- skills/kumo-data-science/hosts/generic/runtime.md | 14 ++++++++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/agent-skills/skills/hosts/generic/runtime.md b/agent-skills/skills/hosts/generic/runtime.md index bb501fbd5..7691085df 100644 --- a/agent-skills/skills/hosts/generic/runtime.md +++ b/agent-skills/skills/hosts/generic/runtime.md @@ -278,6 +278,13 @@ system; the destination may be in another system (`adapter.md` §4) and plays no schema = pa.schema([(keep[f.name], anchor_type(f.type) if keep[f.name] == 'TIMESTAMP' else f.type) for f in scanner.projected_schema]) + + def to_column(col, field): # text in a non-ISO format parses with the entity's timestamp_format + if (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT + and (pa.types.is_string(col.type) or pa.types.is_large_string(col.type))): + return pc.strptime(col, format=ENTITY_TIME_FORMAT, unit=field.type.unit) + return pc.cast(col, field.type) + pop_fs, pop_dir = arrow_fs(POPULATION_PATH) if isinstance(pop_fs, pyarrow.fs.LocalFileSystem): pop_fs.create_dir(pop_dir) # object stores need no directory @@ -285,7 +292,7 @@ system; the destination may be in another system (`adapter.md` §4) and plays no with papq.ParquetWriter(final + '.partial', schema, filesystem=pop_fs) as writer: for batch in scanner.to_batches(): # in batches: bounded memory writer.write_batch(pa.RecordBatch.from_arrays( - [pc.cast(batch[f.name], t.type) for f, t in zip(batch.schema, schema)], + [to_column(batch[f.name], t) for f, t in zip(batch.schema, schema)], schema=schema)) pop_fs.move(final + '.partial', final) # replace only once every batch is in if POPULATION_PATH.startswith('s3://'): # the source's kind, in the §2 forms @@ -300,7 +307,10 @@ system; the destination may be in another system (`adapter.md` §4) and plays no ``` `ENTITY_SOURCE` is the entity table's file or directory as preflight listed it, and - `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. The population is + `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. `ENTITY_TIME_FORMAT` + is the `timestamp_format` the entity `Table` declares on its time column + (`kumoai-reference/references/graph_construction.md`, *A time column must be a real date*), else + `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed it, both in the config cell. Each run replaces its `population.parquet`, as Node 1d says. The diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index 393ab4917..30d73a8c2 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-13-ecf2318d-dirty`** +**`SKILL-BUILD: 2026-11-17-7ace05da-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/hosts/generic/runtime.md b/skills/kumo-data-science/hosts/generic/runtime.md index bb501fbd5..7691085df 100644 --- a/skills/kumo-data-science/hosts/generic/runtime.md +++ b/skills/kumo-data-science/hosts/generic/runtime.md @@ -278,6 +278,13 @@ system; the destination may be in another system (`adapter.md` §4) and plays no schema = pa.schema([(keep[f.name], anchor_type(f.type) if keep[f.name] == 'TIMESTAMP' else f.type) for f in scanner.projected_schema]) + + def to_column(col, field): # text in a non-ISO format parses with the entity's timestamp_format + if (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT + and (pa.types.is_string(col.type) or pa.types.is_large_string(col.type))): + return pc.strptime(col, format=ENTITY_TIME_FORMAT, unit=field.type.unit) + return pc.cast(col, field.type) + pop_fs, pop_dir = arrow_fs(POPULATION_PATH) if isinstance(pop_fs, pyarrow.fs.LocalFileSystem): pop_fs.create_dir(pop_dir) # object stores need no directory @@ -285,7 +292,7 @@ system; the destination may be in another system (`adapter.md` §4) and plays no with papq.ParquetWriter(final + '.partial', schema, filesystem=pop_fs) as writer: for batch in scanner.to_batches(): # in batches: bounded memory writer.write_batch(pa.RecordBatch.from_arrays( - [pc.cast(batch[f.name], t.type) for f, t in zip(batch.schema, schema)], + [to_column(batch[f.name], t) for f, t in zip(batch.schema, schema)], schema=schema)) pop_fs.move(final + '.partial', final) # replace only once every batch is in if POPULATION_PATH.startswith('s3://'): # the source's kind, in the §2 forms @@ -300,7 +307,10 @@ system; the destination may be in another system (`adapter.md` §4) and plays no ``` `ENTITY_SOURCE` is the entity table's file or directory as preflight listed it, and - `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. The population is + `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. `ENTITY_TIME_FORMAT` + is the `timestamp_format` the entity `Table` declares on its time column + (`kumoai-reference/references/graph_construction.md`, *A time column must be a real date*), else + `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed it, both in the config cell. Each run replaces its `population.parquet`, as Node 1d says. The From 1dc2be3fd0252bc7bad81abe1265bfaa35e4be2b Mon Sep 17 00:00:00 2001 From: igutierrez Date: Tue, 17 Nov 2026 11:03:59 -0600 Subject: [PATCH 20/35] agent-skills: publish one directory per skill, as the NVSkills catalog requires The NVSkills catalog does not allow a skill inside another skill, and its components.d schema takes one directory per skill. The published copy nested seven skills inside skills/kumo-data-science/. publish_nvskills.py now lays all eight out side by side under skills/. The package itself is unchanged; only the published layout differs. Links between the skills (../kumo-model-persistence/SKILL.md) already resolve the same way in both layouts. A backticked path that starts with a skill's name now means that skill's directory, nested or beside the router, and the router says so. Two paths that resolved only when nested now name their skill (kumo-data-science/accept/preflight.md, kumo-data-science/hosts//runtime.md). The build checks every reference again in the flat layout. Catalog files move from agent-skills/nvskills/kumo-data-science// to agent-skills/nvskills//. Co-Authored-By: Claude Opus 5.5 --- agent-skills/README.md | 47 ++--- agent-skills/scripts/publish_nvskills.py | 160 +++++++++++++----- .../skills/kumo-data-science/SKILL.md | 6 + .../skills/kumo-graph-visualization/SKILL.md | 2 +- .../references/kumo_finetuning_sdk.md | 2 +- agent-skills/tests/test_publish_nvskills.py | 111 ++++++++---- .../kumo-custom-populations/SKILL.md | 0 .../references/populations.md | 0 skills/kumo-data-science/SKILL.md | 8 +- .../kumo-graph-visualization/SKILL.md | 2 +- .../kumo-model-explanations/SKILL.md | 0 .../references/artifacts.md | 0 .../kumo-model-persistence/SKILL.md | 0 .../references/retention.md | 0 .../references/reuse.md | 0 .../scripts/retain_artifacts.py | 0 .../kumo-model-tuning/SKILL.md | 0 .../kumo-model-tuning/references/plans.md | 0 .../kumo-training-diagnostics/SKILL.md | 0 .../references/diagnostics.md | 0 .../kumoai-reference/SKILL.md | 0 .../references/graph_construction.md | 0 .../references/kumo_finetuning_sdk.md | 2 +- .../kumoai-reference/references/outputs.md | 0 .../references/pql_writing.md | 0 .../references/task_journeys.md | 0 .../references/troubleshooting_signals.md | 0 27 files changed, 239 insertions(+), 101 deletions(-) rename skills/{kumo-data-science => }/kumo-custom-populations/SKILL.md (100%) rename skills/{kumo-data-science => }/kumo-custom-populations/references/populations.md (100%) rename skills/{kumo-data-science => }/kumo-graph-visualization/SKILL.md (97%) rename skills/{kumo-data-science => }/kumo-model-explanations/SKILL.md (100%) rename skills/{kumo-data-science => }/kumo-model-explanations/references/artifacts.md (100%) rename skills/{kumo-data-science => }/kumo-model-persistence/SKILL.md (100%) rename skills/{kumo-data-science => }/kumo-model-persistence/references/retention.md (100%) rename skills/{kumo-data-science => }/kumo-model-persistence/references/reuse.md (100%) rename skills/{kumo-data-science => }/kumo-model-persistence/scripts/retain_artifacts.py (100%) rename skills/{kumo-data-science => }/kumo-model-tuning/SKILL.md (100%) rename skills/{kumo-data-science => }/kumo-model-tuning/references/plans.md (100%) rename skills/{kumo-data-science => }/kumo-training-diagnostics/SKILL.md (100%) rename skills/{kumo-data-science => }/kumo-training-diagnostics/references/diagnostics.md (100%) rename skills/{kumo-data-science => }/kumoai-reference/SKILL.md (100%) rename skills/{kumo-data-science => }/kumoai-reference/references/graph_construction.md (100%) rename skills/{kumo-data-science => }/kumoai-reference/references/kumo_finetuning_sdk.md (99%) rename skills/{kumo-data-science => }/kumoai-reference/references/outputs.md (100%) rename skills/{kumo-data-science => }/kumoai-reference/references/pql_writing.md (100%) rename skills/{kumo-data-science => }/kumoai-reference/references/task_journeys.md (100%) rename skills/{kumo-data-science => }/kumoai-reference/references/troubleshooting_signals.md (100%) diff --git a/agent-skills/README.md b/agent-skills/README.md index 3b832c333..90968d8d4 100644 --- a/agent-skills/README.md +++ b/agent-skills/README.md @@ -130,23 +130,31 @@ so the workflow also runs on changes to `kumo-engine`, `kumo-core`, `kumo-api` a [NVIDIA/skills](https://github.com/NVIDIA/skills) pulls every product's skills from its source repo once an hour. It accepts only content that NVSkills CI has evaluated and signed. NVSkills CI runs -only on PRs that change a root-level `skills/` directory. So the catalog reads a built copy, -`skills/kumo-data-science/` at the repository root, not `agent-skills/skills/`: - -The catalog treats every directory with a `SKILL.md` as a separate skill, and its sync deletes any -such directory that lacks its own `skill.oms.sig`, `skill-card.md` and eval dataset. The package has -eight: the router `skills/kumo-data-science/` and the seven skills nested in it -(`kumoai-reference/` and the six `kumo-*/` capability skills). They stay nested so the router's -relative links resolve, and each of the eight is its own catalog skill: +only on PRs that change a root-level `skills/` directory. So the catalog reads a built copy in +`skills/` at the repository root, not `agent-skills/skills/`. + +The catalog does not allow a skill inside another skill. The package nests seven skills inside the +router (`kumoai-reference/` and the six `kumo-*/` capability skills), so the published copy lays +all eight out side by side: `skills/kumo-data-science/`, `skills/kumoai-reference/`, +`skills/kumo-model-tuning/` and so on, each its own catalog skill. Their links to one another +(`../kumo-model-persistence/SKILL.md`) resolve the same way in both layouts. A backticked path that +starts with a skill's name (`kumoai-reference/references/outputs.md`) names that skill's directory, +as the router's *Paths in these files* says; any other path is relative to the skill that names it. +`publish_nvskills.py` checks every reference in the published layout too, so a path that resolves +only when nested (`accept/preflight.md` written in `kumoai-reference/`) fails the build: write it +`kumo-data-science/accept/preflight.md`. + +The catalog sync deletes any skill directory that lacks its own `skill.oms.sig`, `skill-card.md` +and eval dataset: | Path, in each of the eight skill directories | Written by | |---|---| | everything else | `agent-skills/scripts/publish_nvskills.py`: the `generic` package | -| `skill-card.md`, `evals/` | copied from the same path under `agent-skills/nvskills/kumo-data-science/`, e.g. `agent-skills/nvskills/kumo-data-science/kumo-model-tuning/skill-card.md` | +| `skill-card.md`, `evals/` | copied from `agent-skills/nvskills//`, e.g. `agent-skills/nvskills/kumo-model-tuning/skill-card.md` | | `BENCHMARK.md`, `skill.oms.sig` | NVSkills CI, in your PR; never by hand | Once `agent-skills/nvskills/` exists, the build fails unless every skill has both a card and an -`evals/*.json`, so a new nested skill cannot be published without them. Until then +`evals/*.json`, so a new skill cannot be published without them. Until then `publish_nvskills.py` lists what is missing. Whatever is on `main` is what the catalog publishes. To change the skill: @@ -169,12 +177,11 @@ merge. Run `/nvskills-ci` on reverts too. None of this exists yet; the catalog publishes nothing until all of it does. The card follows the template in NVIDIA/skills `docs/skill-cards.mdx`; the evals follow the `evals/evals.json` of a -published catalog skill. Each skill needs its own, under -`agent-skills/nvskills/kumo-data-science/`: +published catalog skill. Each skill needs its own, under `agent-skills/nvskills/`: | Skill | Card | Evals | |---|---|---| -| `kumo-data-science` (router) | [ ] `skill-card.md` | [ ] `evals/evals.json` | +| `kumo-data-science` (router) | [ ] `kumo-data-science/skill-card.md` | [ ] `kumo-data-science/evals/evals.json` | | `kumoai-reference` | [ ] `kumoai-reference/skill-card.md` | [ ] `kumoai-reference/evals/evals.json` | | `kumo-custom-populations` | [ ] `kumo-custom-populations/skill-card.md` | [ ] `kumo-custom-populations/evals/evals.json` | | `kumo-graph-visualization` | [ ] `kumo-graph-visualization/skill-card.md` | [ ] `kumo-graph-visualization/evals/evals.json` | @@ -183,14 +190,14 @@ published catalog skill. Each skill needs its own, under | `kumo-model-tuning` | [ ] `kumo-model-tuning/skill-card.md` | [ ] `kumo-model-tuning/evals/evals.json` | | `kumo-training-diagnostics` | [ ] `kumo-training-diagnostics/skill-card.md` | [ ] `kumo-training-diagnostics/evals/evals.json` | -- [ ] confirmation from the NVSkills CI maintainers that `/nvskills-ci` evaluates and signs each - of the eight directories, writing `BENCHMARK.md` and `skill.oms.sig` into each. No skill in - the catalog today nests another, so this layout has not been through NVSkills CI before +- [ ] confirmation from the NVSkills CI maintainers that a skill may link to files in a sibling + skill directory (`kumo-data-science/` reads `kumoai-reference/`). Each directory is signed + and published as its own skill - [ ] the `NVSKILLS_CI_DISPATCH_TOKEN` repository secret, from the NVSkills CI maintainers; `request-nvskills-ci.yml` fails without it - [ ] a signed copy on `main`: a PR through steps 1–4 whose `BENCHMARK.md` says PASS in every skill directory -- [ ] the onboarding PR to NVIDIA/skills adding `components.d/kumo-engine.yml`, pointing at - `skills/kumo-data-science`. Its `verify-source-onboarding` check verifies only the paths the - file lists, so list all eight to have each checked. It passes only once the signed copy is - on `main`. +- [ ] the onboarding PR to NVIDIA/skills adding `components.d/kumo-engine.yml`, with one + `skills:` entry per directory: `path: skills//`, `catalog_dir: `, all eight. + Its `verify-source-onboarding` check verifies only the paths the file lists. It passes only + once the signed copy is on `main`. diff --git a/agent-skills/scripts/publish_nvskills.py b/agent-skills/scripts/publish_nvskills.py index 76e2e1da6..31dcf3ce8 100644 --- a/agent-skills/scripts/publish_nvskills.py +++ b/agent-skills/scripts/publish_nvskills.py @@ -2,33 +2,35 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Refresh the copy of the skill that the NVIDIA skills catalog publishes. +"""Refresh the copy of the skills that the NVIDIA skills catalog publishes. - python3 agent-skills/scripts/publish_nvskills.py # rewrite skills/kumo-data-science/ + python3 agent-skills/scripts/publish_nvskills.py # rewrite skills/ python3 agent-skills/scripts/publish_nvskills.py --check # exit 1 if that copy is out of date The NVIDIA/skills catalog (github.com/NVIDIA/skills) pulls skills from each product repo -every hour. It reads them from `components.d/.yml` in the catalog, which names a -directory in this repository. It accepts a skill only if the skill's files match the signature -that NVSkills CI writes. NVSkills CI runs only on pull requests that change `skills/` or +every hour. It reads them from `components.d/.yml` in the catalog, which names one +directory in this repository per skill. It accepts a skill only if the skill's files match the +signature that NVSkills CI writes. NVSkills CI runs only on pull requests that change `skills/` or `team-skills/` at the repository root. `agent-skills/skills/` is the source layout, not a package, so the catalog cannot read it directly. This script writes the `generic` package (the one -for Claude Code, Codex and other coding agents) to `skills/kumo-data-science/`. A PR that changes -the skill runs it and commits the result, and `tests/test_publish_nvskills.py` fails the PR when -it did not. NVSkills CI signs that same PR, so merging it to `main` is what publishes. +for Claude Code, Codex and other coding agents) to `skills/`. A PR that changes the skill runs it +and commits the result, and `tests/test_publish_nvskills.py` fails the PR when it did not. +NVSkills CI signs that same PR, so merging it to `main` is what publishes. -The catalog treats every directory holding a `SKILL.md` as its own skill, and its sync deletes -any such directory that lacks its own `skill.oms.sig`, `skill-card.md` and eval dataset. The -package has eight: the router and the seven skills nested inside it, which stay nested so the -router's relative links resolve. Each of the eight is a catalog skill here. +The catalog does not allow a skill inside another skill. The package nests the seven other skills +inside the router, `kumo-data-science/`, so this script lays them out beside it instead: eight +directories under `skills/`, each holding one `SKILL.md`. Their links to one another resolve the +same way in both layouts (`../kumo-model-persistence/SKILL.md`), and a backticked path that starts +with a skill's name is read from the directory holding the skills, so the build checks every +reference again in the flat layout. -Three kinds of file sit in the published directory: +The catalog sync deletes any skill directory that lacks its own `skill.oms.sig`, `skill-card.md` +and eval dataset. Three kinds of file sit in each published skill directory: * the package, rebuilt from `agent-skills/skills/` on every run; -* the catalog files that NVSkills requires and that we write: each skill's `skill-card.md` and - `evals/*.json`. Their source is `agent-skills/nvskills/kumo-data-science/`, laid out like the - package (a nested skill's files under `/`), and they are copied in on every run. A - catalog file must belong to a skill directory and may not replace a package file. Once that +* the catalog files that NVSkills requires and that we write: the skill's `skill-card.md` and + `evals/*.json`. Their source is `agent-skills/nvskills//`, and they are copied in on + every run. A catalog file must belong to a skill and may not replace a package file. Once that source exists, the build fails unless every skill has both, so the sync never drops one; * `BENCHMARK.md` and `skill.oms.sig`, which NVSkills CI writes into the pull request. They are never edited here, and the build fails if the package or the catalog files carry either @@ -36,6 +38,8 @@ content until NVSkills CI runs again. The catalog then keeps serving the last signed version rather than dropping the skill. +Everything else under `skills/` is this script's output: a refresh deletes it. + `--check` ignores the `SKILL-BUILD` stamp, which names the commit that built the package and so differs on every commit even when no skill file changed. """ @@ -56,9 +60,10 @@ spec.loader.exec_module(package_skill) HOST = 'generic' +ROUTER = package_skill.ROUTER REPO = package_skill.ROOT.parent -PUBLISHED = REPO / 'skills' / package_skill.ROUTER -CATALOG_FILES = package_skill.ROOT / 'nvskills' / package_skill.ROUTER +PUBLISHED = REPO / 'skills' +CATALOG_FILES = package_skill.ROOT / 'nvskills' # Written by NVSkills CI into the pull request; never generated or deleted here. SIGNED = frozenset({'BENCHMARK.md', 'skill.oms.sig'}) # What each skill directory needs from the catalog files; the sync drops a skill without them. @@ -66,37 +71,95 @@ EVALS = 'evals' -def skill_dirs(pkg: Path) -> list[str]: - """Each directory of `pkg` that the catalog treats as a skill, `.` for the router.""" - return sorted(p.parent.relative_to(pkg).as_posix() for p in pkg.rglob('SKILL.md')) +def skill_dirs(root: Path) -> list[str]: + """Each skill directory of the flat layout under `root`, by name.""" + return sorted(p.parent.name for p in root.glob('*/SKILL.md')) + + +def nested(root: Path) -> list[str]: + """Every `SKILL.md` under `root` that is not at the top of its own skill directory.""" + return sorted(p.relative_to(root).as_posix() for p in root.rglob('SKILL.md') + if len(p.relative_to(root).parts) != 2) def owner(rel: Path) -> str | None: - """The skill directory a catalog file belongs to, or None if it is not a catalog input.""" + """The skill a catalog file belongs to, or None if it is not a catalog input.""" parts = rel.parts - if parts[-1] == CARD: - return Path(*parts[:-1]).as_posix() if len(parts) > 1 else '.' - if EVALS in parts[:-1]: - i = parts.index(EVALS) - return Path(*parts[:i]).as_posix() if i else '.' + if len(parts) == 2 and parts[1] == CARD: + return parts[0] + if len(parts) > 2 and parts[1] == EVALS: + return parts[0] return None -def missing(pkg: Path) -> list[str]: - """The catalog inputs a skill directory of `pkg` lacks, as package-relative paths.""" +def missing(root: Path) -> list[str]: + """The catalog inputs a skill directory under `root` lacks, as `root`-relative paths.""" out = [] - for skill in skill_dirs(pkg): - d = pkg / skill + for skill in skill_dirs(root): + d = root / skill if not (d / CARD).is_file(): - out.append((Path(skill) / CARD).as_posix()) + out.append(f'{skill}/{CARD}') if not any((d / EVALS).glob('*.json')): - out.append((Path(skill) / EVALS / '*.json').as_posix()) + out.append(f'{skill}/{EVALS}/*.json') + return out + + +def unresolved(root: Path) -> list[str]: + """Return every reference in the flat layout under `root` that does not resolve. + + A markdown link resolves from the file that holds it. A backticked path resolves from `root` + when it starts with a skill's name, and from its own skill directory when it starts with one + of that skill's entries. A path that starts with another skill's entry would be read from the + wrong directory, so it is an error. + """ + root = root.resolve() + skills = set(skill_dirs(root)) + entries = {s: {p.name for p in (root / s).iterdir()} for s in skills} + errors = [] + for doc in sorted(root.rglob('*.md')): + text = doc.read_text(encoding='utf-8') + where = doc.relative_to(root).as_posix() + skill = doc.relative_to(root).parts[0] + for target in package_skill.MD_LINK.findall(text): + if '://' in target or target.startswith('mailto:'): + continue + resolved = (doc.parent / target).resolve() + if not resolved.is_file() or not resolved.is_relative_to(root): + errors.append(f'{where}: link ({target}) does not resolve') + for target in package_skill.CODE_PATH.findall(text): + head = target.split('/', 1)[0] + if head in skills: + path = root / target + elif head in entries.get(skill, ()): + path = root / skill / target + elif any(head in names for names in entries.values()): + errors.append(f'{where}: `{target}` is another skill\'s file: start it with ' + f'that skill\'s name') + continue + else: + continue # not a package path, e.g. a Kumo source module + if not path.is_file(): + errors.append(f'{where}: `{target}` does not resolve') + return errors + + +def flatten(pkg: Path, out: Path) -> Path: + """Lay the package `pkg` out under `out` with every skill beside the router.""" + inner = sorted(p.parent.relative_to(pkg) for p in pkg.rglob('SKILL.md') if p.parent != pkg) + deep = [p.as_posix() for p in inner if len(p.parts) != 1] + if deep: + raise SystemExit(f'{HOST} package nests skills below its top level: {deep}') + names = {p.name for p in inner} + shutil.copytree(pkg, out / ROUTER, + ignore=lambda d, entries: names & set(entries) if Path(d) == pkg else ()) + for name in sorted(names): + shutil.copytree(pkg / name, out / name) return out def build(out: Path, catalog_files: Path = CATALOG_FILES) -> Path: - """Build the package the catalog publishes into `out` and return its skill directory.""" - pkg = package_skill.build(HOST, out) + """Build the flat layout the catalog publishes into `out` and return it.""" + pkg = package_skill.build(HOST, out / 'package') errors = package_skill.unresolved(pkg) if errors: raise SystemExit(f'{HOST}: {len(errors)} unresolved reference(s): {errors[:3]}') @@ -104,8 +167,13 @@ def build(out: Path, catalog_files: Path = CATALOG_FILES) -> Path: signed = [p.relative_to(pkg).as_posix() for p in pkg.rglob('*') if p.name in SIGNED] if signed: raise SystemExit(f'{HOST} package ships {signed}: written by NVSkills CI, not by hand') + root = flatten(pkg, out / 'skills') + errors = unresolved(root) + if errors: + raise SystemExit(f'{HOST}, one directory per skill: {len(errors)} unresolved ' + f'reference(s): {errors[:3]}') if catalog_files.is_dir(): - skills = skill_dirs(pkg) + skills = skill_dirs(root) for path in sorted(catalog_files.rglob('*')): if path.is_file(): rel = path.relative_to(catalog_files) @@ -114,19 +182,19 @@ def build(out: Path, catalog_files: Path = CATALOG_FILES) -> Path: if owner(rel) not in skills: raise SystemExit(f'{path}: {rel} is not the {CARD} or {EVALS}/ of a skill ' f'in the {HOST} package ({skills})') - target = pkg / rel + target = root / rel if target.exists(): raise SystemExit(f'{path}: {rel} is already in the {HOST} package') target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(path, target) # The catalog sync deletes a skill directory without its own card and evals, and with - # it the files the router links to. Before the first one is written nothing is + # it the files the other skills link to. Before the first one is written nothing is # published, so the gap is reported (`not_ready`) rather than fatal. - gaps = missing(pkg) + gaps = missing(root) if gaps: raise SystemExit(f'{catalog_files}: the catalog would drop these skills; add ' f'{gaps}') - return pkg + return root def not_ready(catalog_files: Path = CATALOG_FILES) -> list[str]: @@ -144,7 +212,7 @@ def files(root: Path) -> dict[str, bytes]: rel = path.relative_to(root).as_posix() if path.is_file() and path.name not in SIGNED: data = path.read_bytes() - if rel == 'SKILL.md': + if rel == f'{ROUTER}/SKILL.md': text = package_skill.MARKER.sub('SKILL-BUILD: x', data.decode('utf-8')) data = text.encode('utf-8') out[rel] = data @@ -162,21 +230,21 @@ def stale(dest: Path = PUBLISHED, catalog_files: Path = CATALOG_FILES) -> list[s def publish(dest: Path = PUBLISHED, catalog_files: Path = CATALOG_FILES) -> list[str]: - """Rewrite `dest` with the current package, keeping the files NVSkills CI wrote.""" + """Rewrite `dest` with the current skills, keeping the files NVSkills CI wrote.""" changes = stale(dest, catalog_files) if not changes: return [] with tempfile.TemporaryDirectory() as tmp: - pkg = build(Path(tmp), catalog_files) + root = build(Path(tmp), catalog_files) if dest.is_dir(): for path in sorted(dest.rglob('*'), reverse=True): if path.is_file() and path.name not in SIGNED: path.unlink() elif path.is_dir() and not any(path.iterdir()): path.rmdir() - for path in sorted(pkg.rglob('*')): + for path in sorted(root.rglob('*')): if path.is_file(): - target = dest / path.relative_to(pkg) + target = dest / path.relative_to(root) target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(path, target) return changes diff --git a/agent-skills/skills/kumo-data-science/SKILL.md b/agent-skills/skills/kumo-data-science/SKILL.md index 8b6cd192c..0668c6cf5 100644 --- a/agent-skills/skills/kumo-data-science/SKILL.md +++ b/agent-skills/skills/kumo-data-science/SKILL.md @@ -16,6 +16,12 @@ Targets the local `kumo.engine` API; the baseline bundle is `20261007+2e1eb12`. Graph visualization requires a source build including [NVIDIA/kumo-engine#134](https://github.com/NVIDIA/kumo-engine/pull/134); it is absent from that bundle and the `1.0.0` release. One SDK, no variants. +## Paths in these files + +A path that starts with a skill's name — `kumo-data-science/`, `kumoai-reference/`, or a skill in +*Routing* below — is in that skill's directory, which sits inside this one or beside it. Any other +path, such as `accept/` or `hosts/`, is relative to the skill whose file names it. + ## Read the host adapter first Before you answer any request this skill covers — a new model, scoring, a request routed to a diff --git a/agent-skills/skills/kumo-graph-visualization/SKILL.md b/agent-skills/skills/kumo-graph-visualization/SKILL.md index 6c29d3ac9..4052954f5 100644 --- a/agent-skills/skills/kumo-graph-visualization/SKILL.md +++ b/agent-skills/skills/kumo-graph-visualization/SKILL.md @@ -60,7 +60,7 @@ Display the graph in a notebook or save the format the user requests: `.svg` for `graph.visualize(EXPORT_PATH, show_cols=True)` also shows ordinary columns in the diagram. `EXPORT_PATH` is the path the user named. When they named none, choose one that does not exist yet and report it; when the named file exists, say so and confirm before replacing it. Use `layout='standard'` or `layout='compact'` to select an arrangement; the default is `'auto'`. -Export paths belong to the Python runtime; use `hosts//runtime.md` for display and file retrieval. +Export paths belong to the Python runtime; use `kumo-data-science/hosts//runtime.md` for display and file retrieval. Keep samples off unless the user requests source values. `show_samples=True` queries source data and includes sample values in saved notebook output; file exports omit them. diff --git a/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md b/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md index f1b0b943b..d464366f0 100644 --- a/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md +++ b/agent-skills/skills/kumoai-reference/references/kumo_finetuning_sdk.md @@ -220,7 +220,7 @@ table names as `table_names()` returns them and column names as `.columns` retur **Never exclude a column by a case-sensitive `!=` against a hand-typed literal.** A filter such as `c.name != 'END_DATE'` silently keeps a lower-case `end_date` — the comparison never matches, the column is never dropped, and nothing raises. Pass the confirmed `Excluded columns` from preflight -(`accept/preflight.md` §8) as a literal `column_names=` list built from the catalog's own spelling +(`kumo-data-science/accept/preflight.md` §8) as a literal `column_names=` list built from the catalog's own spelling — it has no hand-typed comparison to get wrong. If a filter is used instead, compare case-insensitively: `c.name.lower() != 'end_date'.lower()`. diff --git a/agent-skills/tests/test_publish_nvskills.py b/agent-skills/tests/test_publish_nvskills.py index 25496335e..f0ff99e70 100644 --- a/agent-skills/tests/test_publish_nvskills.py +++ b/agent-skills/tests/test_publish_nvskills.py @@ -15,8 +15,9 @@ spec.loader.exec_module(publish_nvskills) -# The router and every skill the generic package nests inside it. -SKILLS = ['.', 'kumoai-reference', *publish_nvskills.package_skill.CAPABILITY] +# The router and every skill the generic package nests inside it, each published beside it. +ROUTER = publish_nvskills.ROUTER +SKILLS = [ROUTER, 'kumoai-reference', *publish_nvskills.package_skill.CAPABILITY] @pytest.fixture @@ -31,11 +32,11 @@ def catalog(tmp_path: Path) -> Path: def test_publish_writes_the_generic_package_and_catalog_files(tmp_path: Path, catalog: Path) -> None: - dest = tmp_path / 'skills' / 'kumo-data-science' + dest = tmp_path / 'skills' assert publish_nvskills.publish(dest, catalog) - assert (dest / 'SKILL.md').is_file() - assert (dest / 'hosts' / 'generic' / 'adapter.md').is_file() - assert not (dest / 'hosts' / 'snowflake-coco').exists() + assert (dest / ROUTER / 'SKILL.md').is_file() + assert (dest / ROUTER / 'hosts' / 'generic' / 'adapter.md').is_file() + assert not (dest / ROUTER / 'hosts' / 'snowflake-coco').exists() assert sorted(publish_nvskills.skill_dirs(dest)) == sorted(SKILLS) for skill in SKILLS: assert (dest / skill / 'evals' / 'evals.json').is_file() @@ -44,16 +45,59 @@ def test_publish_writes_the_generic_package_and_catalog_files(tmp_path: Path, assert publish_nvskills.stale(dest, catalog) == [] +def test_no_skill_is_published_inside_another(tmp_path: Path, catalog: Path) -> None: + """The catalog does not allow a skill inside a skill: each SKILL.md has its own directory.""" + dest = tmp_path / 'skills' + publish_nvskills.publish(dest, catalog) + assert publish_nvskills.nested(dest) == [] + assert not (dest / ROUTER / 'kumoai-reference').exists() + assert (dest / 'kumoai-reference' / 'references' / 'kumo_finetuning_sdk.md').is_file() + assert publish_nvskills.unresolved(dest) == [] + + +def test_the_package_cannot_nest_a_skill_two_levels_down(tmp_path: Path, catalog: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + build = publish_nvskills.package_skill.build + + def deeper(host: str, dist: Path) -> Path: + pkg = build(host, dist) + (pkg / 'kumo-model-tuning' / 'extra').mkdir() + (pkg / 'kumo-model-tuning' / 'extra' / 'SKILL.md').write_text('---\nname: extra\n---\n') + return pkg + + monkeypatch.setattr(publish_nvskills.package_skill, 'build', deeper) + with pytest.raises(SystemExit, match='nests skills below its top level'): + publish_nvskills.publish(tmp_path / 'skills', catalog) + + +@pytest.mark.parametrize('text, error', [ + ('read `accept/preflight.md`', "is another skill's file"), + ('read `kumoai-reference/references/absent.md`', 'does not resolve'), + ('read `references/absent.md`', 'does not resolve'), + ('see [plans](../kumo-model-tuning/references/absent.md)', 'does not resolve'), +]) +def test_references_are_checked_one_directory_per_skill(tmp_path: Path, catalog: Path, + text: str, error: str) -> None: + """A path a nested package resolves can miss once the skills sit side by side.""" + dest = tmp_path / 'skills' + publish_nvskills.publish(dest, catalog) + doc = dest / 'kumo-model-tuning' / 'references' / 'plans.md' + doc.write_text(doc.read_text() + f'\n{text}\n') + errors = publish_nvskills.unresolved(dest) + assert len(errors) == 1 and error in errors[0], errors + + @pytest.mark.parametrize('rel', ['kumoai-reference/skill-card.md', - 'kumo-model-tuning/evals/evals.json', 'evals/evals.json']) + 'kumo-model-tuning/evals/evals.json', + 'kumo-data-science/evals/evals.json']) def test_every_skill_directory_needs_its_own_card_and_evals(tmp_path: Path, catalog: Path, rel: str) -> None: """The catalog sync deletes a SKILL.md directory without its own card and evals. - For a nested skill that also breaks the router's links into it, so the build refuses. + That also breaks the other skills' links into it, so the build refuses. """ (catalog / rel).unlink() - dest = tmp_path / 'kumo-data-science' + dest = tmp_path / 'skills' with pytest.raises(SystemExit, match='the catalog would drop these skills'): publish_nvskills.publish(dest, catalog) assert not dest.exists() @@ -61,63 +105,70 @@ def test_every_skill_directory_needs_its_own_card_and_evals(tmp_path: Path, cata def test_without_catalog_files_every_input_is_reported_missing(tmp_path: Path) -> None: absent = tmp_path / 'no-catalog' - dest = tmp_path / 'kumo-data-science' + dest = tmp_path / 'skills' assert publish_nvskills.publish(dest, absent) todo = publish_nvskills.not_ready(absent) for skill in SKILLS: - assert (Path(skill) / 'skill-card.md').as_posix() in todo - assert (Path(skill) / 'evals' / '*.json').as_posix() in todo + assert f'{skill}/skill-card.md' in todo + assert f'{skill}/evals/*.json' in todo assert len(todo) == 2 * len(SKILLS) -@pytest.mark.parametrize('rel', ['notes.md', 'SKILL.md', 'hosts/generic/adapter.md', - 'hosts/skill-card.md', 'hosts/evals/evals.json', +@pytest.mark.parametrize('rel', ['notes.md', 'skill-card.md', 'evals/evals.json', + 'kumo-data-science/SKILL.md', + 'kumo-data-science/hosts/generic/adapter.md', + 'kumo-data-science/hosts/skill-card.md', + 'kumo-data-science/hosts/evals/evals.json', + 'kumo-data-science/kumo-model-tuning/skill-card.md', 'kumo-model-tunning/skill-card.md']) def test_catalog_files_must_belong_to_a_skill(tmp_path: Path, catalog: Path, rel: str) -> None: (catalog / rel).parent.mkdir(parents=True, exist_ok=True) (catalog / rel).write_text('stray\n') with pytest.raises(SystemExit, match='is not the skill-card.md or evals/ of a skill'): - publish_nvskills.publish(tmp_path / 'kumo-data-science', catalog) + publish_nvskills.publish(tmp_path / 'skills', catalog) def test_publish_keeps_signed_files_and_drops_files_no_longer_shipped(tmp_path: Path, catalog: Path) -> None: - dest = tmp_path / 'kumo-data-science' + dest = tmp_path / 'skills' publish_nvskills.publish(dest, catalog) signed = [dest / skill / name for skill in SKILLS for name in publish_nvskills.SIGNED] for path in signed: path.write_text('from NVSkills CI\n') - (dest / 'old' / 'removed.md').parent.mkdir() - (dest / 'old' / 'removed.md').write_text('gone\n') + # The nested layout this directory held before it was split into one directory per skill. + (dest / ROUTER / 'kumoai-reference').mkdir() + (dest / ROUTER / 'kumoai-reference' / 'SKILL.md').write_text('gone\n') - assert publish_nvskills.stale(dest, catalog) == ['- old/removed.md'] + assert publish_nvskills.stale(dest, catalog) == [f'- {ROUTER}/kumoai-reference/SKILL.md'] publish_nvskills.publish(dest, catalog) - assert not (dest / 'old').exists() + assert not (dest / ROUTER / 'kumoai-reference').exists() for path in signed: assert path.read_text() == 'from NVSkills CI\n' def test_check_ignores_the_build_stamp_but_not_content(tmp_path: Path, catalog: Path) -> None: - dest = tmp_path / 'kumo-data-science' + dest = tmp_path / 'skills' publish_nvskills.publish(dest, catalog) - router = dest / 'SKILL.md' + router = dest / ROUTER / 'SKILL.md' text = router.read_text(encoding='utf-8') router.write_text(publish_nvskills.package_skill.MARKER.sub('SKILL-BUILD: 2000-01-01-0', text), encoding='utf-8') assert publish_nvskills.stale(dest, catalog) == [] router.write_text(text + '\nedited by hand\n', encoding='utf-8') - assert publish_nvskills.stale(dest, catalog) == ['~ SKILL.md'] + assert publish_nvskills.stale(dest, catalog) == [f'~ {ROUTER}/SKILL.md'] -@pytest.mark.parametrize('rel', ['skill.oms.sig', 'evals/skill.oms.sig', 'evals/BENCHMARK.md', +@pytest.mark.parametrize('rel', ['kumo-data-science/skill.oms.sig', + 'kumo-data-science/evals/skill.oms.sig', + 'kumo-data-science/evals/BENCHMARK.md', 'kumoai-reference/skill.oms.sig', 'kumo-model-tuning/BENCHMARK.md']) def test_catalog_files_cannot_supply_signed_files(tmp_path: Path, catalog: Path, rel: str) -> None: (catalog / rel).write_text('forged\n') with pytest.raises(SystemExit, match='written by NVSkills CI'): - publish_nvskills.publish(tmp_path / 'kumo-data-science', catalog) + publish_nvskills.publish(tmp_path / 'skills', catalog) @pytest.mark.parametrize('rel', ['skill-card.md', 'kumo-model-tuning/evals/evals.json']) @@ -134,7 +185,7 @@ def shipping(host: str, dist: Path) -> Path: monkeypatch.setattr(publish_nvskills.package_skill, 'build', shipping) with pytest.raises(SystemExit, match='already in the generic package'): - publish_nvskills.publish(tmp_path / 'kumo-data-science', catalog) + publish_nvskills.publish(tmp_path / 'skills', catalog) def test_the_package_cannot_ship_signed_files(tmp_path: Path, catalog: Path, @@ -147,7 +198,7 @@ def with_signature(host: str, dist: Path) -> Path: return pkg monkeypatch.setattr(publish_nvskills.package_skill, 'build', with_signature) - dest = tmp_path / 'kumo-data-science' + dest = tmp_path / 'skills' with pytest.raises(SystemExit, match='written by NVSkills CI'): publish_nvskills.publish(dest, catalog) assert not dest.exists() @@ -156,11 +207,11 @@ def with_signature(host: str, dist: Path) -> Path: def test_the_published_copy_matches_the_source() -> None: """The copy NVIDIA/skills syncs from main is rebuilt in the same PR as the change. - A PR that edits agent-skills/ without refreshing skills/kumo-data-science/ would merge a - published copy that no longer matches its source, so it fails here instead. + A PR that edits agent-skills/ without refreshing skills/ would merge a published copy that + no longer matches its source, so it fails here instead. """ changes = publish_nvskills.stale() assert changes == [], ( - 'skills/kumo-data-science/ is out of date; run ' + 'skills/ is out of date; run ' f'python3 agent-skills/scripts/publish_nvskills.py and commit the result: {changes}' ) diff --git a/skills/kumo-data-science/kumo-custom-populations/SKILL.md b/skills/kumo-custom-populations/SKILL.md similarity index 100% rename from skills/kumo-data-science/kumo-custom-populations/SKILL.md rename to skills/kumo-custom-populations/SKILL.md diff --git a/skills/kumo-data-science/kumo-custom-populations/references/populations.md b/skills/kumo-custom-populations/references/populations.md similarity index 100% rename from skills/kumo-data-science/kumo-custom-populations/references/populations.md rename to skills/kumo-custom-populations/references/populations.md diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index 30d73a8c2..0d491be97 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-17-7ace05da-dirty`** +**`SKILL-BUILD: 2026-11-17-2ab10b0f-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. @@ -16,6 +16,12 @@ Targets the local `kumo.engine` API; the baseline bundle is `20261007+2e1eb12`. Graph visualization requires a source build including [NVIDIA/kumo-engine#134](https://github.com/NVIDIA/kumo-engine/pull/134); it is absent from that bundle and the `1.0.0` release. One SDK, no variants. +## Paths in these files + +A path that starts with a skill's name — `kumo-data-science/`, `kumoai-reference/`, or a skill in +*Routing* below — is in that skill's directory, which sits inside this one or beside it. Any other +path, such as `accept/` or `hosts/`, is relative to the skill whose file names it. + ## Read the host adapter first Before you answer any request this skill covers — a new model, scoring, a request routed to a diff --git a/skills/kumo-data-science/kumo-graph-visualization/SKILL.md b/skills/kumo-graph-visualization/SKILL.md similarity index 97% rename from skills/kumo-data-science/kumo-graph-visualization/SKILL.md rename to skills/kumo-graph-visualization/SKILL.md index 6c29d3ac9..4052954f5 100644 --- a/skills/kumo-data-science/kumo-graph-visualization/SKILL.md +++ b/skills/kumo-graph-visualization/SKILL.md @@ -60,7 +60,7 @@ Display the graph in a notebook or save the format the user requests: `.svg` for `graph.visualize(EXPORT_PATH, show_cols=True)` also shows ordinary columns in the diagram. `EXPORT_PATH` is the path the user named. When they named none, choose one that does not exist yet and report it; when the named file exists, say so and confirm before replacing it. Use `layout='standard'` or `layout='compact'` to select an arrangement; the default is `'auto'`. -Export paths belong to the Python runtime; use `hosts//runtime.md` for display and file retrieval. +Export paths belong to the Python runtime; use `kumo-data-science/hosts//runtime.md` for display and file retrieval. Keep samples off unless the user requests source values. `show_samples=True` queries source data and includes sample values in saved notebook output; file exports omit them. diff --git a/skills/kumo-data-science/kumo-model-explanations/SKILL.md b/skills/kumo-model-explanations/SKILL.md similarity index 100% rename from skills/kumo-data-science/kumo-model-explanations/SKILL.md rename to skills/kumo-model-explanations/SKILL.md diff --git a/skills/kumo-data-science/kumo-model-explanations/references/artifacts.md b/skills/kumo-model-explanations/references/artifacts.md similarity index 100% rename from skills/kumo-data-science/kumo-model-explanations/references/artifacts.md rename to skills/kumo-model-explanations/references/artifacts.md diff --git a/skills/kumo-data-science/kumo-model-persistence/SKILL.md b/skills/kumo-model-persistence/SKILL.md similarity index 100% rename from skills/kumo-data-science/kumo-model-persistence/SKILL.md rename to skills/kumo-model-persistence/SKILL.md diff --git a/skills/kumo-data-science/kumo-model-persistence/references/retention.md b/skills/kumo-model-persistence/references/retention.md similarity index 100% rename from skills/kumo-data-science/kumo-model-persistence/references/retention.md rename to skills/kumo-model-persistence/references/retention.md diff --git a/skills/kumo-data-science/kumo-model-persistence/references/reuse.md b/skills/kumo-model-persistence/references/reuse.md similarity index 100% rename from skills/kumo-data-science/kumo-model-persistence/references/reuse.md rename to skills/kumo-model-persistence/references/reuse.md diff --git a/skills/kumo-data-science/kumo-model-persistence/scripts/retain_artifacts.py b/skills/kumo-model-persistence/scripts/retain_artifacts.py similarity index 100% rename from skills/kumo-data-science/kumo-model-persistence/scripts/retain_artifacts.py rename to skills/kumo-model-persistence/scripts/retain_artifacts.py diff --git a/skills/kumo-data-science/kumo-model-tuning/SKILL.md b/skills/kumo-model-tuning/SKILL.md similarity index 100% rename from skills/kumo-data-science/kumo-model-tuning/SKILL.md rename to skills/kumo-model-tuning/SKILL.md diff --git a/skills/kumo-data-science/kumo-model-tuning/references/plans.md b/skills/kumo-model-tuning/references/plans.md similarity index 100% rename from skills/kumo-data-science/kumo-model-tuning/references/plans.md rename to skills/kumo-model-tuning/references/plans.md diff --git a/skills/kumo-data-science/kumo-training-diagnostics/SKILL.md b/skills/kumo-training-diagnostics/SKILL.md similarity index 100% rename from skills/kumo-data-science/kumo-training-diagnostics/SKILL.md rename to skills/kumo-training-diagnostics/SKILL.md diff --git a/skills/kumo-data-science/kumo-training-diagnostics/references/diagnostics.md b/skills/kumo-training-diagnostics/references/diagnostics.md similarity index 100% rename from skills/kumo-data-science/kumo-training-diagnostics/references/diagnostics.md rename to skills/kumo-training-diagnostics/references/diagnostics.md diff --git a/skills/kumo-data-science/kumoai-reference/SKILL.md b/skills/kumoai-reference/SKILL.md similarity index 100% rename from skills/kumo-data-science/kumoai-reference/SKILL.md rename to skills/kumoai-reference/SKILL.md diff --git a/skills/kumo-data-science/kumoai-reference/references/graph_construction.md b/skills/kumoai-reference/references/graph_construction.md similarity index 100% rename from skills/kumo-data-science/kumoai-reference/references/graph_construction.md rename to skills/kumoai-reference/references/graph_construction.md diff --git a/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md b/skills/kumoai-reference/references/kumo_finetuning_sdk.md similarity index 99% rename from skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md rename to skills/kumoai-reference/references/kumo_finetuning_sdk.md index f1b0b943b..d464366f0 100644 --- a/skills/kumo-data-science/kumoai-reference/references/kumo_finetuning_sdk.md +++ b/skills/kumoai-reference/references/kumo_finetuning_sdk.md @@ -220,7 +220,7 @@ table names as `table_names()` returns them and column names as `.columns` retur **Never exclude a column by a case-sensitive `!=` against a hand-typed literal.** A filter such as `c.name != 'END_DATE'` silently keeps a lower-case `end_date` — the comparison never matches, the column is never dropped, and nothing raises. Pass the confirmed `Excluded columns` from preflight -(`accept/preflight.md` §8) as a literal `column_names=` list built from the catalog's own spelling +(`kumo-data-science/accept/preflight.md` §8) as a literal `column_names=` list built from the catalog's own spelling — it has no hand-typed comparison to get wrong. If a filter is used instead, compare case-insensitively: `c.name.lower() != 'end_date'.lower()`. diff --git a/skills/kumo-data-science/kumoai-reference/references/outputs.md b/skills/kumoai-reference/references/outputs.md similarity index 100% rename from skills/kumo-data-science/kumoai-reference/references/outputs.md rename to skills/kumoai-reference/references/outputs.md diff --git a/skills/kumo-data-science/kumoai-reference/references/pql_writing.md b/skills/kumoai-reference/references/pql_writing.md similarity index 100% rename from skills/kumo-data-science/kumoai-reference/references/pql_writing.md rename to skills/kumoai-reference/references/pql_writing.md diff --git a/skills/kumo-data-science/kumoai-reference/references/task_journeys.md b/skills/kumoai-reference/references/task_journeys.md similarity index 100% rename from skills/kumo-data-science/kumoai-reference/references/task_journeys.md rename to skills/kumoai-reference/references/task_journeys.md diff --git a/skills/kumo-data-science/kumoai-reference/references/troubleshooting_signals.md b/skills/kumoai-reference/references/troubleshooting_signals.md similarity index 100% rename from skills/kumo-data-science/kumoai-reference/references/troubleshooting_signals.md rename to skills/kumoai-reference/references/troubleshooting_signals.md From 432514dd0523b464ada1b4681cbc95be218f3771 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Tue, 17 Nov 2026 12:10:30 -0600 Subject: [PATCH 21/35] agent-skills: skill cards and evals for the NVSkills catalog The catalog sync drops a skill directory without its own skill-card.md and evals/*.json, and NVSkills CI runs those evals to write BENCHMARK.md. Each of the eight published skills now has both under agent-skills/nvskills//, and publish_nvskills.py copies them into skills/. - Cards: rendered with NVIDIA/Trustworthy-AI's skill-card-generator template and passing its validate_submission.py. Apache 2.0 from LICENSE, NVIDIA owner. No skill version exists, so the card says the copy is published from main and identified by its skill.oms.sig. - Evals: 45 cases in the catalog's evals.json schema. Each assertion is behaviour the skill's own files document; each set covers the core workflow, a refusal or routing case, and one negative case (expected_skill: null). None needs Kumo, a warehouse or credentials: they check what the agent says or writes, not executed results. - test_no_environment_setup also scans agent-skills/nvskills/, since those files are published beside the skills. Co-Authored-By: Claude Opus 5.5 --- agent-skills/README.md | 42 ++++--- .../kumo-custom-populations/evals/evals.json | 72 +++++++++++ .../kumo-custom-populations/skill-card.md | 45 +++++++ .../kumo-data-science/evals/evals.json | 116 ++++++++++++++++++ .../nvskills/kumo-data-science/skill-card.md | 50 ++++++++ .../kumo-graph-visualization/evals/evals.json | 67 ++++++++++ .../kumo-graph-visualization/skill-card.md | 44 +++++++ .../kumo-model-explanations/evals/evals.json | 81 ++++++++++++ .../kumo-model-explanations/skill-card.md | 45 +++++++ .../kumo-model-persistence/evals/evals.json | 83 +++++++++++++ .../kumo-model-persistence/skill-card.md | 47 +++++++ .../kumo-model-tuning/evals/evals.json | 68 ++++++++++ .../nvskills/kumo-model-tuning/skill-card.md | 45 +++++++ .../evals/evals.json | 66 ++++++++++ .../kumo-training-diagnostics/skill-card.md | 45 +++++++ .../kumoai-reference/evals/evals.json | 77 ++++++++++++ .../nvskills/kumoai-reference/skill-card.md | 50 ++++++++ .../tests/test_no_environment_setup.py | 14 ++- .../kumo-custom-populations/evals/evals.json | 72 +++++++++++ skills/kumo-custom-populations/skill-card.md | 45 +++++++ skills/kumo-data-science/SKILL.md | 2 +- skills/kumo-data-science/evals/evals.json | 116 ++++++++++++++++++ skills/kumo-data-science/skill-card.md | 50 ++++++++ .../kumo-graph-visualization/evals/evals.json | 67 ++++++++++ skills/kumo-graph-visualization/skill-card.md | 44 +++++++ .../kumo-model-explanations/evals/evals.json | 81 ++++++++++++ skills/kumo-model-explanations/skill-card.md | 45 +++++++ .../kumo-model-persistence/evals/evals.json | 83 +++++++++++++ skills/kumo-model-persistence/skill-card.md | 47 +++++++ skills/kumo-model-tuning/evals/evals.json | 68 ++++++++++ skills/kumo-model-tuning/skill-card.md | 45 +++++++ .../evals/evals.json | 66 ++++++++++ .../kumo-training-diagnostics/skill-card.md | 45 +++++++ skills/kumoai-reference/evals/evals.json | 77 ++++++++++++ skills/kumoai-reference/skill-card.md | 50 ++++++++ 35 files changed, 2036 insertions(+), 24 deletions(-) create mode 100644 agent-skills/nvskills/kumo-custom-populations/evals/evals.json create mode 100644 agent-skills/nvskills/kumo-custom-populations/skill-card.md create mode 100644 agent-skills/nvskills/kumo-data-science/evals/evals.json create mode 100644 agent-skills/nvskills/kumo-data-science/skill-card.md create mode 100644 agent-skills/nvskills/kumo-graph-visualization/evals/evals.json create mode 100644 agent-skills/nvskills/kumo-graph-visualization/skill-card.md create mode 100644 agent-skills/nvskills/kumo-model-explanations/evals/evals.json create mode 100644 agent-skills/nvskills/kumo-model-explanations/skill-card.md create mode 100644 agent-skills/nvskills/kumo-model-persistence/evals/evals.json create mode 100644 agent-skills/nvskills/kumo-model-persistence/skill-card.md create mode 100644 agent-skills/nvskills/kumo-model-tuning/evals/evals.json create mode 100644 agent-skills/nvskills/kumo-model-tuning/skill-card.md create mode 100644 agent-skills/nvskills/kumo-training-diagnostics/evals/evals.json create mode 100644 agent-skills/nvskills/kumo-training-diagnostics/skill-card.md create mode 100644 agent-skills/nvskills/kumoai-reference/evals/evals.json create mode 100644 agent-skills/nvskills/kumoai-reference/skill-card.md create mode 100644 skills/kumo-custom-populations/evals/evals.json create mode 100644 skills/kumo-custom-populations/skill-card.md create mode 100644 skills/kumo-data-science/evals/evals.json create mode 100644 skills/kumo-data-science/skill-card.md create mode 100644 skills/kumo-graph-visualization/evals/evals.json create mode 100644 skills/kumo-graph-visualization/skill-card.md create mode 100644 skills/kumo-model-explanations/evals/evals.json create mode 100644 skills/kumo-model-explanations/skill-card.md create mode 100644 skills/kumo-model-persistence/evals/evals.json create mode 100644 skills/kumo-model-persistence/skill-card.md create mode 100644 skills/kumo-model-tuning/evals/evals.json create mode 100644 skills/kumo-model-tuning/skill-card.md create mode 100644 skills/kumo-training-diagnostics/evals/evals.json create mode 100644 skills/kumo-training-diagnostics/skill-card.md create mode 100644 skills/kumoai-reference/evals/evals.json create mode 100644 skills/kumoai-reference/skill-card.md diff --git a/agent-skills/README.md b/agent-skills/README.md index 90968d8d4..8c642ae0d 100644 --- a/agent-skills/README.md +++ b/agent-skills/README.md @@ -175,29 +175,35 @@ merge. Run `/nvskills-ci` on reverts too. ### Before the first publish -None of this exists yet; the catalog publishes nothing until all of it does. The card follows the -template in NVIDIA/skills `docs/skill-cards.mdx`; the evals follow the `evals/evals.json` of a -published catalog skill. Each skill needs its own, under `agent-skills/nvskills/`: +The catalog publishes nothing until all of this is done. Each skill has its own card and evals +under `agent-skills/nvskills/`. The cards are rendered with the catalog's `skill-card-generator` +(template in NVIDIA/skills `docs/skill-cards.mdx`) and pass its `validate_submission.py`. The +evals use the `evals/evals.json` schema of published catalog skills, and assert only behaviour the +skill's own files document. Each set has one negative case (`expected_skill: null`). | Skill | Card | Evals | |---|---|---| -| `kumo-data-science` (router) | [ ] `kumo-data-science/skill-card.md` | [ ] `kumo-data-science/evals/evals.json` | -| `kumoai-reference` | [ ] `kumoai-reference/skill-card.md` | [ ] `kumoai-reference/evals/evals.json` | -| `kumo-custom-populations` | [ ] `kumo-custom-populations/skill-card.md` | [ ] `kumo-custom-populations/evals/evals.json` | -| `kumo-graph-visualization` | [ ] `kumo-graph-visualization/skill-card.md` | [ ] `kumo-graph-visualization/evals/evals.json` | -| `kumo-model-explanations` | [ ] `kumo-model-explanations/skill-card.md` | [ ] `kumo-model-explanations/evals/evals.json` | -| `kumo-model-persistence` | [ ] `kumo-model-persistence/skill-card.md` | [ ] `kumo-model-persistence/evals/evals.json` | -| `kumo-model-tuning` | [ ] `kumo-model-tuning/skill-card.md` | [ ] `kumo-model-tuning/evals/evals.json` | -| `kumo-training-diagnostics` | [ ] `kumo-training-diagnostics/skill-card.md` | [ ] `kumo-training-diagnostics/evals/evals.json` | - +| `kumo-data-science` (router) | [x] `kumo-data-science/skill-card.md` | [x] `kumo-data-science/evals/evals.json` | +| `kumoai-reference` | [x] `kumoai-reference/skill-card.md` | [x] `kumoai-reference/evals/evals.json` | +| `kumo-custom-populations` | [x] `kumo-custom-populations/skill-card.md` | [x] `kumo-custom-populations/evals/evals.json` | +| `kumo-graph-visualization` | [x] `kumo-graph-visualization/skill-card.md` | [x] `kumo-graph-visualization/evals/evals.json` | +| `kumo-model-explanations` | [x] `kumo-model-explanations/skill-card.md` | [x] `kumo-model-explanations/evals/evals.json` | +| `kumo-model-persistence` | [x] `kumo-model-persistence/skill-card.md` | [x] `kumo-model-persistence/evals/evals.json` | +| `kumo-model-tuning` | [x] `kumo-model-tuning/skill-card.md` | [x] `kumo-model-tuning/evals/evals.json` | +| `kumo-training-diagnostics` | [x] `kumo-training-diagnostics/skill-card.md` | [x] `kumo-training-diagnostics/evals/evals.json` | + +- [x] the `NVSKILLS_CI_DISPATCH_TOKEN` repository secret, from the NVSkills CI maintainers; + `request-nvskills-ci.yml` fails without it +- [x] kumo-engine registered with NVSkills CI (NVIDIA/nvskills-ci#112: `main`, external profile, + no fork PRs) and the `nv-skills-ci` GitHub App installed +- [x] the catalog PR to NVIDIA/skills adding `components.d/kumo-engine.yml`, with one `skills:` + entry per directory (`path: skills//`, `catalog_dir: `), all eight: + NVIDIA/skills#885, a draft. Its `verify-source-onboarding` check verifies only the paths the + file lists, and passes only once the signed copy is on `main` +- [ ] `request-nvskills-ci.yml` on `main` (NVIDIA/kumo-engine#371). `/nvskills-ci` is an + `issue_comment` trigger, so it runs only from `main`'s copy - [ ] confirmation from the NVSkills CI maintainers that a skill may link to files in a sibling skill directory (`kumo-data-science/` reads `kumoai-reference/`). Each directory is signed and published as its own skill -- [ ] the `NVSKILLS_CI_DISPATCH_TOKEN` repository secret, from the NVSkills CI maintainers; - `request-nvskills-ci.yml` fails without it - [ ] a signed copy on `main`: a PR through steps 1–4 whose `BENCHMARK.md` says PASS in every skill directory -- [ ] the onboarding PR to NVIDIA/skills adding `components.d/kumo-engine.yml`, with one - `skills:` entry per directory: `path: skills//`, `catalog_dir: `, all eight. - Its `verify-source-onboarding` check verifies only the paths the file lists. It passes only - once the signed copy is on `main`. diff --git a/agent-skills/nvskills/kumo-custom-populations/evals/evals.json b/agent-skills/nvskills/kumo-custom-populations/evals/evals.json new file mode 100644 index 000000000..45adaf00d --- /dev/null +++ b/agent-skills/nvskills/kumo-custom-populations/evals/evals.json @@ -0,0 +1,72 @@ +{ + "skill_name": "kumo-custom-populations", + "evals": [ + { + "id": "kpop-01", + "prompt": "Train my confirmed Kumo churn query on our own labels instead of generated ones. The Snowflake table ML.LABELS.CURATED_LABELS has ENTITY1, TARGET1, TIMESTAMP, WEIGHT and SPLIT (0 train, 1 validation, 2 test). `pq` and `snapshot` are already built in the notebook.", + "expected_output": "Before any code it replies with the plan and stops for confirmation: the table and its fields, WEIGHT and SPLIT as the weight and split columns, what it validated, and the constructor `kumo.TrainingTable.from_source_table(population_connector['CURATED_LABELS'], TrainingTableSpec(weight_col='WEIGHT', split_col='SPLIT'), pq=pq, snapshot=snapshot)`, reusing the confirmed pq and snapshot, with `suggest_model_plan(..., train_table_spec=spec)`.", + "assertions": [ + "Names kumo.TrainingTable.from_source_table as the constructor", + "Uses TrainingTableSpec with weight_col='WEIGHT' and split_col='SPLIT'", + "Reuses the existing pq and snapshot", + "Passes the same spec to suggest_model_plan as train_table_spec", + "Presents the plan and stops for the user's confirmation before training" + ], + "expected_skill": "kumo-custom-populations", + "expected_script": null + }, + { + "id": "kpop-02", + "prompt": "Our Kumo labels table ML.LABELS.CURATED_LABELS has a SPLIT_LABEL column with the values 'train', 'val' and 'test'. Use it as the split column and train.", + "expected_output": "Reports SPLIT_LABEL as the offending field: the split column reads only 0, 1 and 2, so string labels match no split and their rows are skipped (the build fails with 'custom-TT source kept no rows' if none remain). It shows the labels and their counts from a read-only query and asks for either a prepared table with 0/1/2 or explicit approval of a shown mapping, which would then be applied in a read-only connector.sql(...) source. It writes nothing back.", + "assertions": [ + "Says the split column must hold 0, 1 or 2 and that 'train'/'val'/'test' will not match", + "Explains that unmatched rows are skipped, so training could keep no rows", + "Asks for a prepared 0/1/2 table or explicit approval of a mapping it shows", + "Proposes the mapping only as a read-only connector.sql(...) source, after approval", + "Does not UPDATE the table, CREATE TABLE AS, or write a derived table" + ], + "expected_skill": "kumo-custom-populations", + "expected_script": null + }, + { + "id": "kpop-03", + "prompt": "For my Kumo custom training table, just normalize the WEIGHT column to mean 1 and multiply positives by 10. You have my approval, go ahead.", + "expected_output": "An approval given before the formula and distribution are shown is not approval. It states the exact formula and the columns it reads, shows the resulting weight distribution (min, mean, max, counts per label) from a read-only query, and stops for explicit approval. Only then would it apply the formula in a read-only connector.sql(...) source; it never writes to the source table.", + "assertions": [ + "Does not apply the weight change straight away despite the up-front approval", + "Shows the exact weight formula and the columns it reads", + "Shows or proposes a read-only query for the resulting weight distribution", + "Stops for explicit approval after showing them", + "Does not UPDATE the source table or write a derived table" + ], + "expected_skill": "kumo-custom-populations", + "expected_script": null + }, + { + "id": "kpop-04", + "prompt": "Score exactly the accounts in ML.EVAL.EVAL_ACCOUNTS (ACCOUNT_ID, ANCHOR_TS, CHURNED) with my fitted Kumo trainer, so I can compare against CHURNED afterwards.", + "expected_output": "The scoring population must not carry the target, so the table cannot be passed as is. Evaluation labels (CHURNED) stay separate and are joined to the scores afterwards. It proposes, for approval, a read-only connector.sql(...) source keeping only the entity and TIMESTAMP columns under their contract names, or asks for a table without the target, and scores with `kumo.PredictionTable.from_source_table(..., pq=pq, snapshot=snapshot, plan=trainer.training_table_generation_plan)` then `trainer.predict(population, snapshot=snapshot, ...)`, comparing output entity and anchor coverage with the population.", + "assertions": [ + "Says the scoring population must not contain the target column CHURNED", + "Keeps CHURNED separate for joining to the scores afterwards", + "Proposes a read-only connector.sql(...) source with the entity and timestamp columns, or asks for a table without the target", + "Uses kumo.PredictionTable.from_source_table with pq and snapshot", + "Says output rows or coverage will be compared with the population rather than assumed equal" + ], + "expected_skill": "kumo-custom-populations", + "expected_script": null + }, + { + "id": "kpop-05", + "prompt": "Split my pandas DataFrame into stratified train, validation and test sets (70/15/15) with scikit-learn.", + "expected_output": "A scikit-learn data-splitting task unrelated to Kumo. The agent answers with train_test_split, without loading the Kumo custom-populations skill.", + "assertions": [ + "Does not load or follow the kumo-custom-populations skill", + "Uses scikit-learn train_test_split with stratification to produce 70/15/15 splits" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/agent-skills/nvskills/kumo-custom-populations/skill-card.md b/agent-skills/nvskills/kumo-custom-populations/skill-card.md new file mode 100644 index 000000000..8a1a901ae --- /dev/null +++ b/agent-skills/nvskills/kumo-custom-populations/skill-card.md @@ -0,0 +1,45 @@ +## Description:
+Train with caller-supplied labels, weights and splits or score exact entity/anchor rows through Snowflake source tables using the local Kumo API.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents when generated Kumo training or prediction tables do not represent the rows they need, for example curated labels, manual splits, sample weights or an exact scoring population.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Yes]
+**Credential Type(s):** [Other [Snowflake connection or Snowpark session of the user's notebook]]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/populations.md](references/populations.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python and SQL code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Reads source tables only through read-only queries; never writes back to them.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/agent-skills/nvskills/kumo-data-science/evals/evals.json b/agent-skills/nvskills/kumo-data-science/evals/evals.json new file mode 100644 index 000000000..a82cdf78a --- /dev/null +++ b/agent-skills/nvskills/kumo-data-science/evals/evals.json @@ -0,0 +1,116 @@ +{ + "skill_name": "kumo-data-science", + "evals": [ + { + "id": "kds-01", + "prompt": "Use my sales data to figure out which customers are risky. Build me a Kumo model for it.", + "expected_output": "Treats this as a `train` request with required fields missing. In one ask-back it asks for where the data lives (for example a database and schema, or a folder of files), the observable outcome that makes a customer 'risky' (the positive-label condition), and the prediction window. It does not ask for columns, keys, joins or an output table, does not guess an outcome, and generates no code.", + "assertions": [ + "Asks where the data lives, as a location such as a database and schema, a dataset, a bucket prefix or a folder of files", + "Asks for a concrete, observable outcome that defines 'risky' (the positive-label condition) instead of inventing one", + "Asks for the prediction window (for example 'in the next 30 days')", + "Asks for all missing fields together in a single reply", + "Does not ask for column names, primary keys, joins or an output table", + "Does not write a notebook, Kumo SDK code or a predictive query" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-02", + "prompt": "Create a notebook that trains a Kumo model for our fraud use case. The data is in Snowflake, USER_DATA.E_COMMERCE. Entity: each customer. Outcome: at least one fraud event. Window: next 30 days. You can't reach our Snowflake account from here, so this is the structure; no keys are declared:\n\nCUSTOMERS(CUSTOMER_ID NUMBER, COUNTRY VARCHAR, SIGNUP_AT TIMESTAMP_NTZ)\nFRAUD_EVENTS(EVENT_ID NUMBER, CUSTOMER_ID NUMBER, EVENT_AT TIMESTAMP_NTZ, AMOUNT NUMBER, RESOLVED_AT TIMESTAMP_NTZ, DAYS_TO_RESOLVE NUMBER)\nTRANSACTIONS(TRANSACTION_ID NUMBER, CUSTOMER_ID NUMBER, CREATED_AT TIMESTAMP_NTZ, AMOUNT NUMBER)", + "expected_output": "All required fields are present, so it builds the graph proposal from the pasted structure and stops for confirmation. The Source scope line says the structure came from the user. Keys CUSTOMERS.CUSTOMER_ID, FRAUD_EVENTS.EVENT_ID and TRANSACTIONS.TRANSACTION_ID are proposed by name and, since no probe can run, tagged as unverified; time columns are EVENT_AT and CREATED_AT; FRAUD_EVENTS.CUSTOMER_ID and TRANSACTIONS.CUSTOMER_ID link to CUSTOMERS.CUSTOMER_ID; table mode all-connected [default]. An Excluded columns section removes FRAUD_EVENTS.RESOLVED_AT and DAYS_TO_RESOLVE as not knowable at the row's EVENT_AT. It ends by asking the user to confirm or correct any line, with no Kumo code.", + "assertions": [ + "Presents a graph proposal (metadata, links, table mode) built from the pasted structure rather than asking for the structure again", + "Tags proposal values with provenance tags such as [supplied], [inferred], [inferred, unverified] or [default]", + "Says the structure came from the user rather than from a read of the source", + "Proposes links from FRAUD_EVENTS.CUSTOMER_ID and TRANSACTIONS.CUSTOMER_ID to CUSTOMERS.CUSTOMER_ID", + "Lists FRAUD_EVENTS.RESOLVED_AT and FRAUD_EVENTS.DAYS_TO_RESOLVE as excluded columns because their values are not known at the event time", + "Asks the user to confirm or correct the proposal and stops there", + "Does not emit Kumo SDK code or a predictive query in this reply" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-03", + "prompt": "Build a Kumo notebook that predicts, for each account in our BigQuery dataset acme-prod:billing, whether the account will have at least one failed payment in the next 14 days.", + "expected_output": "The request has every required field. Before preflight it checks whether it can read the dataset's columns in this session (a BigQuery tool, an authenticated `bq` on the PATH, or Python with Kumo and ambient credentials). It cannot, so it sends one ask-back saying what it checked and what happened, and offers two ways forward: the user makes a read tool available, or pastes the structure (tables, columns, types, declared keys). It never asks for a credential value, does not sign in or change tool configuration, does not invent tables, and writes no code.", + "assertions": [ + "Says what it checked to read the dataset and what the result was", + "Offers the user a way to grant read access (an authenticated tool or a connected tool for BigQuery)", + "Offers the alternative of pasting the structure: tables, columns with types and any declared keys", + "Does not ask for a password, key, token or other credential value in the chat", + "Does not invent tables, columns or a graph proposal for acme-prod:billing", + "Does not re-ask for the entity, outcome or window, which the user already gave", + "Does not generate the notebook in this reply" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-04", + "prompt": "Earlier in this session you proposed this graph for our fraud request (Snowflake USER_DATA.E_COMMERCE; entity: each customer; outcome: at least one fraud event; window: next 30 days):\n\nMetadata\n- CUSTOMERS: primary key CUSTOMER_ID [inferred]\n- FRAUD_EVENTS: primary key EVENT_ID [inferred]; time column EVENT_AT [inferred]\nLinks\n- FRAUD_EVENTS.CUSTOMER_ID -> CUSTOMERS.CUSTOMER_ID [inferred]\nExcluded columns\n- FRAUD_EVENTS.RESOLVED_AT [inferred]\n\nI confirm the graph. Please write the predictions to ANALYTICS.ML.fraud_scores.", + "expected_output": "With the graph confirmed it moves to the task interpretation gate and restates the task in the documented shape: one prediction per CUSTOMERS.CUSTOMER_ID; outcome at least one fraud event; window 0 to 30 days after the prediction anchor; temporal binary classification; predictions written to ANALYTICS.ML.fraud_scores, verbatim, in the same notebook; threshold 0.5 [default]. It says whether the destination already exists and would be replaced, or that it could not check. It then asks for explicit confirmation and emits no Kumo code.", + "assertions": [ + "Restates the entity as one prediction per customer (CUSTOMERS.CUSTOMER_ID)", + "Restates the outcome as at least one fraud event and the window as the next 30 days", + "Names the task type as temporal binary classification", + "Carries the destination ANALYTICS.ML.fraud_scores verbatim, without changing its case or adding a suffix", + "States a classification threshold of 0.5 as the default", + "Says whether the destination table already exists and would be replaced on every run, or that it could not check", + "Asks for explicit confirmation and does not emit Kumo SDK code or a notebook yet" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-05", + "prompt": "I trained a Kumo model in my notebook an hour ago and `training_result` is still in memory. Which columns influenced its predictions the most?", + "expected_output": "This is an attribution question about an existing run, not a new training request. It routes to the kumo-model-explanations skill (or says that skill is not available here), keeps the existing run, and does not run new-training intake or start another fit.", + "assertions": [ + "Treats the request as an explanation question about the existing run, handled by the kumo-model-explanations skill (or reports that skill is unavailable rather than improvising its workflow)", + "Does not ask for the data location, entity, outcome or prediction window", + "Does not start, propose or generate a new training run" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-06", + "prompt": "Schedule the Kumo churn notebook you generated to run every night at 2am as a job on our cluster, and tell me when the first run finishes.", + "expected_output": "In this generic coding-agent host there is no job file: the skill does not submit, schedule or monitor jobs. It says so and offers the notebook or script to run themselves. It does not create a scheduler entry or submit anything.", + "assertions": [ + "Says that in this environment the skill does not submit, schedule or monitor jobs", + "Offers the notebook or a script as what it can provide instead", + "Does not create a cron entry, scheduler configuration or job submission", + "Does not claim to watch or report on a run" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-07", + "prompt": "Set up kumo.init with my KUMO_API_KEY and train a churn model on Kumo's hosted service.", + "expected_output": "Hosted Kumo (kumo.init, API keys, KUMO_API_KEY / KUMO_API_ENDPOINT, Kumo Studio) is not the SDK this skill targets. It says so and stops, and does not blend hosted setup into generated code.", + "assertions": [ + "Says that hosted Kumo (kumo.init with an API key) is not the SDK this skill works with", + "Does not write code that calls kumo.init or reads KUMO_API_KEY", + "Does not ask the user to paste an API key" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-08", + "prompt": "Write a pandas function that removes duplicate rows from a DataFrame, keeping the most recent row per order_id by updated_at.", + "expected_output": "A plain pandas task with no Kumo or predictive-model element. The agent answers directly, for example with sort_values plus drop_duplicates, without loading the Kumo skill or starting its intake.", + "assertions": [ + "Does not load or follow the kumo-data-science skill or ask Kumo intake questions", + "Provides pandas code that keeps the latest row per order_id by updated_at" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/agent-skills/nvskills/kumo-data-science/skill-card.md b/agent-skills/nvskills/kumo-data-science/skill-card.md new file mode 100644 index 000000000..9d4179590 --- /dev/null +++ b/agent-skills/nvskills/kumo-data-science/skill-card.md @@ -0,0 +1,50 @@ +## Description:
+Turns a plain-language request for a KumoAI predictive model on relational data into a confirmed task and graph through a multi-turn intake, generates one notebook or script that trains and predicts with the local Kumo SDK, and routes inspection, persistence, population and tuning requests to their own Kumo skills.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to build Kumo models for churn, propensity, fraud, recommendation, forecasting and link prediction on data in Snowflake, BigQuery, Databricks, S3, GCS or local files: the agent checks the request, proposes the graph for confirmation and writes the training-and-prediction notebook.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Optional]
+**Credential Type(s):** [Cloud Credentials, OAuth Token, Other [Snowflake connections.toml entry]]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [accept/intake_contract.md](accept/intake_contract.md)
+- [accept/preflight.md](accept/preflight.md)
+- [accept/batch_inference.md](accept/batch_inference.md)
+- [validate/atomic_question_check.md](validate/atomic_question_check.md)
+- [hosts/generic/adapter.md](hosts/generic/adapter.md)
+- [hosts/generic/runtime.md](hosts/generic/runtime.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown replies (ask-backs, graph proposal, task restatement) and one Jupyter notebook or Python script]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Generates code only; runs it only when the user asks. Writes predictions only to a destination the user confirmed.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/agent-skills/nvskills/kumo-graph-visualization/evals/evals.json b/agent-skills/nvskills/kumo-graph-visualization/evals/evals.json new file mode 100644 index 000000000..0be558c67 --- /dev/null +++ b/agent-skills/nvskills/kumo-graph-visualization/evals/evals.json @@ -0,0 +1,67 @@ +{ + "skill_name": "kumo-graph-visualization", + "evals": [ + { + "id": "kviz-01", + "prompt": "Add a graph diagram step to my Kumo training script churn_30d_train.py.", + "expected_output": "Adds, after `graph.validate()` and before snapshot and training, a step that saves the diagram next to the script as `churn_30d_train.graph.svg` via `Path(__file__).with_suffix('.graph.svg')`, guarded by `hasattr(graph, 'visualize')` and a try/except that prints the error and lets training continue, and prints the saved path.", + "assertions": [ + "Saves the diagram with graph.visualize to a path derived from the script name, churn_30d_train.graph.svg", + "Places the step after graph validation and before snapshot creation or training", + "Guards with hasattr(graph, 'visualize') and a try/except that lets the script continue", + "Prints or reports the saved diagram path" + ], + "expected_skill": "kumo-graph-visualization", + "expected_script": null + }, + { + "id": "kviz-02", + "prompt": "Export my Kumo graph to reports/graph.pdf. That file already exists from last week.", + "expected_output": "A view/export request needs no training intake. `.pdf` gives a diagram plus a complete schema index. Because the named file exists, it says so and asks for confirmation before replacing it, then uses `graph.visualize('reports/graph.pdf')`. Samples stay off.", + "assertions": [ + "Does not ask model-training intake questions", + "Uses graph.visualize with the .pdf path", + "Says the file already exists and asks before replacing it", + "Does not turn on show_samples" + ], + "expected_skill": "kumo-graph-visualization", + "expected_script": null + }, + { + "id": "kviz-03", + "prompt": "I'm deciding which table to add to my Kumo graph next. Give me a text view of all its columns, keys and relationships that you can read.", + "expected_output": "Exports the graph to Markdown in a new temporary directory (`graph.visualize(schema_path)` with a `.md` path), reads that file for columns, key and time roles and relationships, and notes the schema does not measure predictive usefulness.", + "assertions": [ + "Exports the graph to a .md file with graph.visualize", + "Writes it to a new temporary directory so nothing is replaced", + "Reads the exported file to inspect columns, keys and relationships", + "Says the schema view does not measure how useful a table would be for prediction" + ], + "expected_skill": "kumo-graph-visualization", + "expected_script": null + }, + { + "id": "kviz-04", + "prompt": "Show my Kumo graph in the notebook. It has 45 tables, and employees.manager_id links back to employees.employee_id.", + "expected_output": "The viewer supports up to 30 tables and does not support self-relations, so this graph cannot be fully shown. It reports that and lets modeling continue with the original graph, without changing the graph to make it render.", + "assertions": [ + "Says the viewer supports at most 30 tables", + "Says self-relations such as employees.manager_id -> employees are not supported", + "Does not remove tables or relationships from the model's graph to make it render" + ], + "expected_skill": "kumo-graph-visualization", + "expected_script": null + }, + { + "id": "kviz-05", + "prompt": "Draw my social network from edges.csv (source, target) with networkx and matplotlib.", + "expected_output": "A networkx plotting task unrelated to Kumo. The agent answers with networkx code without loading the Kumo graph-visualization skill.", + "assertions": [ + "Does not load or follow the kumo-graph-visualization skill", + "Uses networkx to read edges.csv and matplotlib to draw it" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/agent-skills/nvskills/kumo-graph-visualization/skill-card.md b/agent-skills/nvskills/kumo-graph-visualization/skill-card.md new file mode 100644 index 000000000..9eed168ad --- /dev/null +++ b/agent-skills/nvskills/kumo-graph-visualization/skill-card.md @@ -0,0 +1,44 @@ +## Description:
+Provide graph visuals in Kumo notebooks and SDK scripts, display or export relationships on request, and inspect Markdown schemas during model iteration.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to view a Kumo graph's tables and relationships in a notebook, export it as SVG, PDF or Markdown, and read its schema while deciding on graph changes.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks; SVG, PDF or Markdown graph files when exported]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Source sample values are off unless the user asks for them.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/agent-skills/nvskills/kumo-model-explanations/evals/evals.json b/agent-skills/nvskills/kumo-model-explanations/evals/evals.json new file mode 100644 index 000000000..08061a6c7 --- /dev/null +++ b/agent-skills/nvskills/kumo-model-explanations/evals/evals.json @@ -0,0 +1,81 @@ +{ + "skill_name": "kumo-model-explanations", + "evals": [ + { + "id": "kexpl-01", + "prompt": "Which columns influenced my Kumo model the most? I fit it in this notebook and `training_result` is still in memory.", + "expected_output": "Uses `training_result.explanations`, checks `availability['gradient']`, and reads `gradient_scores().to_pandas()` (optionally `plot_gradient`). Names the result Kumo gradient attribution over the processed TEST subset, normalized across the graph, and not drivers, probabilities, percentages or causal effects. Reports the run, stage and file paths, and notes the files are local scratch excluded from the published model bundle. Does not retrain.", + "assertions": [ + "Uses training_result.explanations and checks availability before reading", + "Reads gradient scores with gradient_scores() or plot_gradient()", + "Calls the scores Kumo gradient attribution over the processed test subset", + "Says the scores are not causal effects, probabilities or percentages", + "Reports the run and the TEST stage the scores come from", + "Does not start another fit" + ], + "expected_skill": "kumo-model-explanations", + "expected_script": null + }, + { + "id": "kexpl-02", + "prompt": "For my Kumo model, compare predictions with actual outcomes across age cohorts. `training_result` is in memory.", + "expected_output": "Uses column analysis: `explanations.column_analysis()`, `cohort_keys()` to find the (table, hop, column) key, then `cohort(key)` / `plot_cohort(key)`. Reports each cohort's population share beside its mean prediction and, when recorded, mean target. A cohort mismatch is a lead to investigate, not proof of leakage or causation; prediction_score and target_score are not AUROC or error metrics.", + "assertions": [ + "Uses column analysis with cohort_keys() and cohort() or plot_cohort()", + "Reports population share alongside mean prediction and mean target per cohort", + "Says a prediction-versus-target mismatch is a lead, not proof of leakage or a cause", + "Does not treat prediction_score or target_score as AUROC or an error metric" + ], + "expected_skill": "kumo-model-explanations", + "expected_script": null + }, + { + "id": "kexpl-03", + "prompt": "I reloaded my Kumo model with `Trainer.load(bundle_dir)` and there are no explanations. Just call `trainer.explain()` to regenerate them.", + "expected_output": "There is no public `Trainer.explain()` or regeneration API. Published bundles exclude explanation files, so a loaded bundle reports them unavailable (`model_only`). The retained training run can be reopened with `kumo.TrainingExplanations.load(run_dir)`. It says no fit was started and asks whether the user wants one.", + "assertions": [ + "Says there is no public Trainer.explain() to regenerate explanations", + "Says the model bundle does not contain explanation files", + "Suggests reopening the retained run with kumo.TrainingExplanations.load(run_dir)", + "Does not start a fit, and asks before proposing one" + ], + "expected_skill": "kumo-model-explanations", + "expected_script": null + }, + { + "id": "kexpl-04", + "prompt": "Run column analysis and radiant attribution on my Kumo link-prediction model.", + "expected_output": "Kumo calls the method gradient attribution, not Radiant. Column analysis is restricted to supported node-level tasks and is not available for link prediction. Gradient scoring may exist, but TWO_TOWER link models are skipped, so availability must be checked.", + "assertions": [ + "Says Kumo calls the method gradient attribution", + "Says column analysis is not available for link prediction", + "Says gradient attribution availability must be checked for this link model (TWO_TOWER models are skipped)" + ], + "expected_skill": "kumo-model-explanations", + "expected_script": null + }, + { + "id": "kexpl-05", + "prompt": "There are three Kumo training runs under /scratch/kumo/runs. Show me the gradient attribution.", + "expected_output": "Lists the candidate runs with their paths and asks which one the user means before showing any score. It never picks the newest directory or combines runs.", + "assertions": [ + "Lists the candidate runs with their paths", + "Asks which run the user means before showing scores", + "Does not pick the newest run or combine runs" + ], + "expected_skill": "kumo-model-explanations", + "expected_script": null + }, + { + "id": "kexpl-06", + "prompt": "Compute SHAP values for my XGBoost classifier and show a beeswarm plot.", + "expected_output": "A SHAP task on an XGBoost model, unrelated to Kumo. The agent uses the shap library without loading the Kumo model-explanations skill.", + "assertions": [ + "Does not load or follow the kumo-model-explanations skill", + "Uses the shap library to compute values and draw a beeswarm plot" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/agent-skills/nvskills/kumo-model-explanations/skill-card.md b/agent-skills/nvskills/kumo-model-explanations/skill-card.md new file mode 100644 index 000000000..e231160f8 --- /dev/null +++ b/agent-skills/nvskills/kumo-model-explanations/skill-card.md @@ -0,0 +1,45 @@ +## Description:
+Inspect and interpret gradient attribution, column analysis and sampled explanation subgraphs from a Kumo model trained in a Python notebook.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to read the explanation artifacts of an existing Kumo run, such as which table columns the model relied on and how predictions compare with outcomes across column cohorts, without retraining.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/artifacts.md](references/artifacts.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown assessment with inline Python code blocks and plots]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Inspection only; makes no causal claims and starts no fit.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/agent-skills/nvskills/kumo-model-persistence/evals/evals.json b/agent-skills/nvskills/kumo-model-persistence/evals/evals.json new file mode 100644 index 000000000..6dcdc4911 --- /dev/null +++ b/agent-skills/nvskills/kumo-model-persistence/evals/evals.json @@ -0,0 +1,83 @@ +{ + "skill_name": "kumo-model-persistence", + "evals": [ + { + "id": "kpersist-01", + "prompt": "Train the confirmed Kumo churn model (model_plan, training_table and snapshot are ready in my Snowflake notebook) and publish it to the model registry in ML.REGISTRY as CHURN_MODEL version V3.", + "expected_output": "Publishes during the fit: `trainer.fit(training_table, snapshot=snapshot, export_config=kumo.ModelOutputConfig(connector=registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V3')))` with a connector scoped to ML.REGISTRY, then checks `training_result.published_model` and raises if it is None. Uses `location`, not `path`, for Snowflake. Checks that V3 does not already exist first and does not overwrite an existing version.", + "assertions": [ + "Passes a kumo.ModelOutputConfig as export_config to trainer.fit", + "Uses kumo.ModelRegistryLocation with model_name CHURN_MODEL and version V3", + "Uses location= rather than path= for the Snowflake registry", + "Checks training_result.published_model and treats None as a failure", + "Checks that version V3 does not already exist and does not overwrite an existing version" + ], + "expected_skill": "kumo-model-persistence", + "expected_script": null + }, + { + "id": "kpersist-02", + "prompt": "Load CHURN_MODEL from our Snowflake model registry in ML.REGISTRY and score new customers. SHOW VERSIONS lists V1, V2 and V3.", + "expected_output": "Several versions exist and the user named none, so it asks which version to load rather than choosing one. It explains that `version=None` loads the registry default, which is not necessarily the newest, and that 'latest' would mean the newest created_on, passed explicitly. It does not retrain.", + "assertions": [ + "Asks which version to load instead of picking one", + "Explains that loading without a version uses the registry default, not necessarily the newest", + "Does not train a new model" + ], + "expected_skill": "kumo-model-persistence", + "expected_script": null + }, + { + "id": "kpersist-03", + "prompt": "Load version V2 of CHURN_MODEL from the Snowflake registry ML.REGISTRY and score our customers as of 2026-12-01. The graph-building code and query are in our training record for V2.", + "expected_output": "Restores with `kumo.Trainer.load_from_connector(registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V2'))`, rebuilds the graph from the V2 training record (not the live schema), creates a fresh `graph.snapshot()`, generates the prediction table with a naive `prediction_time` and `plan=loaded.training_table_generation_plan`, and predicts with `snapshot=` and an explicit binary threshold. No retraining.", + "assertions": [ + "Loads with kumo.Trainer.load_from_connector and an explicit ModelRegistryLocation for version V2", + "Rebuilds the graph from the V2 training record rather than from the live schema or another version", + "Builds a new graph.snapshot() for scoring", + "Passes plan=loaded.training_table_generation_plan to generate_prediction_table", + "Passes snapshot= to predict and an explicit binary classification threshold", + "Does not call fit" + ], + "expected_skill": "kumo-model-persistence", + "expected_script": null + }, + { + "id": "kpersist-04", + "prompt": "Before this Snowflake notebook shuts down, keep my Kumo run's aggregate metrics and the per-trial validation results on the internal stage @ML.ARTIFACTS so I can look at them next week.", + "expected_output": "Selects explicit files from one run (not the whole scratch tree, which can hold source data and credentials), writes aggregate metrics to a new JSON file with allow_nan=False, builds an archive with the packaged `create_archive` helper from an explicit list of relative paths, uploads with `session.file.put(..., auto_compress=False, overwrite=False)` to a new run-specific prefix and checks exactly one `UPLOADED` result, and records the stage path and archive SHA256 digest.", + "assertions": [ + "Selects explicit files from one run instead of uploading the whole scratch directory", + "Uses the packaged create_archive helper with a list of explicit relative file paths", + "Uploads with session.file.put and overwrite=False", + "Checks that the upload status is UPLOADED and treats anything else, including SKIPPED, as not retained", + "Records the stage object path and the archive digest" + ], + "expected_skill": "kumo-model-persistence", + "expected_script": null + }, + { + "id": "kpersist-05", + "prompt": "Look up my Kumo model by its hosted job ID job-8841 and redeploy it.", + "expected_output": "The local Kumo API has no hosted model or job ID lookup. It says so and asks for the registry model name and version, or a retained bundle, instead of inventing a lookup.", + "assertions": [ + "Says there is no hosted model or job ID lookup in this local API", + "Asks for a registry model and explicit version, or a retained bundle", + "Does not invent an API call that fetches a model by job ID" + ], + "expected_skill": "kumo-model-persistence", + "expected_script": null + }, + { + "id": "kpersist-06", + "prompt": "Save my trained scikit-learn model with joblib and upload it to s3://ml-models/fraud/.", + "expected_output": "A scikit-learn persistence task unrelated to Kumo. The agent answers with joblib and an S3 upload, without loading the Kumo persistence skill.", + "assertions": [ + "Does not load or follow the kumo-model-persistence skill", + "Uses joblib to save the model and uploads it to the S3 path" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/agent-skills/nvskills/kumo-model-persistence/skill-card.md b/agent-skills/nvskills/kumo-model-persistence/skill-card.md new file mode 100644 index 000000000..1b02d2b88 --- /dev/null +++ b/agent-skills/nvskills/kumo-model-persistence/skill-card.md @@ -0,0 +1,47 @@ +## Description:
+Publish and reload Kumo models through the Snowflake registry, reconstruct compatible scoring inputs, and retain or retrieve run diagnostics and output files through a Snowflake stage.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to publish a Kumo model during its fit, reload an explicit registry version or retained bundle for scoring, and keep selected run files on a Snowflake stage after the notebook session ends.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Yes]
+**Credential Type(s):** [Other [Snowflake connection or Snowpark session of the user's notebook]]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/reuse.md](references/reuse.md)
+- [references/retention.md](references/retention.md)
+- [scripts/retain_artifacts.py](scripts/retain_artifacts.py)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks; an archive with a manifest of byte lengths and SHA256 digests when artifacts are retained]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Loading a model does not authorize training, replacing a registry version or overwriting a destination.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/agent-skills/nvskills/kumo-model-tuning/evals/evals.json b/agent-skills/nvskills/kumo-model-tuning/evals/evals.json new file mode 100644 index 000000000..2b5419d26 --- /dev/null +++ b/agent-skills/nvskills/kumo-model-tuning/evals/evals.json @@ -0,0 +1,68 @@ +{ + "skill_name": "kumo-model-tuning", + "evals": [ + { + "id": "ktune-01", + "prompt": "My Kumo `trainer` is already fitted in this notebook with the plan I tuned last week. Before a long run on new data I want a quick smoke test: one experiment, three epochs, no refits. Show me the plan changes.", + "expected_output": "Starts from the existing plan (`trainer.model_plan`), not a fresh `suggest_model_plan`, which would reset settings the user did not mention. Applies only the requested changes: `training_job.num_experiments = 1`, `training_job.refit_trainval = False`, `training_job.refit_full = False`, `optimization.max_epochs = 3`. Says this is a limited functional check, not a production-quality setting, and presents the resolved plan without running the fit.", + "assertions": [ + "Starts from the existing plan (trainer.model_plan) rather than a new suggest_model_plan", + "Sets plan.training_job.num_experiments = 1", + "Sets plan.training_job.refit_trainval = False and plan.training_job.refit_full = False", + "Sets plan.optimization.max_epochs = 3", + "Says the smoke test is a limited functional check, not a production-quality configuration", + "Does not run fit as part of presenting the plan" + ], + "expected_skill": "kumo-model-tuning", + "expected_script": null + }, + { + "id": "ktune-02", + "prompt": "In my existing Kumo model plan, use a numerical encoder for customers.account_balance. The plan already has encoder overrides for two other columns.", + "expected_output": "Imports `Numerical` from `kumoapi.encoder` and adds `'customers.account_balance': Numerical()` to `plan.column_processing.encoder_overrides`, merging into the existing overrides rather than replacing them. Notes that the key must match the graph's actual table.column and semantic type.", + "assertions": [ + "Uses Numerical from kumoapi.encoder", + "Sets the override under plan.column_processing.encoder_overrides with the key 'customers.account_balance'", + "Merges into the existing overrides instead of replacing the whole mapping", + "Notes the key must match the graph's table and column (and its semantic type)" + ], + "expected_skill": "kumo-model-tuning", + "expected_script": null + }, + { + "id": "ktune-03", + "prompt": "Set forecast_length = 4 in my Kumo model plan so the model forecasts four periods ahead.", + "expected_output": "Explains that `forecast_length` belongs to `TrainingTableGenerationPlan`, not `ModelPlan`, together with generation, split and lag settings, so it cannot be set on the model plan. Does not invent a model-plan field for it.", + "assertions": [ + "Says forecast_length is not a ModelPlan setting", + "Says it belongs to TrainingTableGenerationPlan", + "Does not set a forecast_length field on the model plan" + ], + "expected_skill": "kumo-model-tuning", + "expected_script": null + }, + { + "id": "ktune-04", + "prompt": "I want my Kumo model tuned on a custom Python metric I wrote, `def profit_at_k(y, p): ...`. Plug it into the model plan.", + "expected_output": "Says that arbitrary Python metric callables are not supported: metric names go in `plan.training_job.metrics` and `tune_metric`, chosen from those the task supports, with the tuning direction kept explicit. It may suggest a supported metric but does not wire the function in.", + "assertions": [ + "Says a custom Python metric function cannot be used by the model plan", + "Says metrics are chosen by name in plan.training_job.metrics and tune_metric", + "Does not write code that passes the Python function to the plan" + ], + "expected_skill": "kumo-model-tuning", + "expected_script": null + }, + { + "id": "ktune-05", + "prompt": "Tune the hyperparameters of my scikit-learn RandomForestClassifier with GridSearchCV over n_estimators and max_depth.", + "expected_output": "A scikit-learn task unrelated to Kumo. The agent writes GridSearchCV code without loading the Kumo model-tuning skill.", + "assertions": [ + "Does not load or follow the kumo-model-tuning skill", + "Provides GridSearchCV code over n_estimators and max_depth" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/agent-skills/nvskills/kumo-model-tuning/skill-card.md b/agent-skills/nvskills/kumo-model-tuning/skill-card.md new file mode 100644 index 000000000..9f36b8483 --- /dev/null +++ b/agent-skills/nvskills/kumo-model-tuning/skill-card.md @@ -0,0 +1,45 @@ +## Description:
+Configure a Kumo model plan for a user-defined task, including encoders, run mode, bounded experiments, sampling, optimization, architecture, metrics and refits.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to change a Kumo model plan for a confirmed task, such as a smoke test, encoder overrides or bounded experiments, while keeping the target and evaluation protocol unchanged.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/plans.md](references/plans.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Suggesting a plan does not run its experiments.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/agent-skills/nvskills/kumo-training-diagnostics/evals/evals.json b/agent-skills/nvskills/kumo-training-diagnostics/evals/evals.json new file mode 100644 index 000000000..641ff73e1 --- /dev/null +++ b/agent-skills/nvskills/kumo-training-diagnostics/evals/evals.json @@ -0,0 +1,66 @@ +{ + "skill_name": "kumo-training-diagnostics", + "evals": [ + { + "id": "kdiag-01", + "prompt": "My Kumo churn model just finished training in this notebook and `training_result` is in memory. How did it do?", + "expected_output": "Reads `training_result.metrics` as an attribute and lists the metric keys and finite values that are actually present, treating missing values as unavailable, not zero. For binary classification it explains that AUROC measures ranking and AP/AUPRC depends on prevalence. It does not assume metrics that were not computed and does not retrain or tune.", + "assertions": [ + "Reads training_result.metrics as an attribute, not a method call", + "Reports only the metric keys actually present, treating missing values as unavailable rather than zero", + "Explains that AUROC measures ranking quality and AP/AUPRC depends on class prevalence", + "Does not start a new fit or tuning run" + ], + "expected_skill": "kumo-training-diagnostics", + "expected_script": null + }, + { + "id": "kdiag-02", + "prompt": "Is my Kumo model overfitting? `training_result` from the fit is still in memory; look at the learning curves.", + "expected_output": "Locates the run from `training_result.bundle_dir`, finds `auto_train_progress.json` under it, and reads each trial's epoch records from `trial_progress`, keeping trials and refits separate. A widening train/validation gap can suggest overfitting, but distribution shift, split design and sampling can also explain it. It does not join unrelated trials into one curve and does not retrain.", + "assertions": [ + "Finds the run directory from training_result.bundle_dir and reads auto_train_progress.json", + "Reads per-trial epoch metrics and keeps trial and refit histories separate", + "Says a widening train/validation gap can suggest overfitting but has other possible explanations", + "Does not plot unrelated trials as one continuous curve", + "Does not start a new fit" + ], + "expected_skill": "kumo-training-diagnostics", + "expected_script": null + }, + { + "id": "kdiag-03", + "prompt": "Pick the Kumo trial with the best test AUROC from my last run and use that one instead of the one Kumo selected.", + "expected_output": "Declines to select by test-set performance. Trial selection uses the configured validation tuning metric and its direction, recorded in `best_trainer.json`; the agent should not replace the SDK's selection with its own ranking. Test results are held for final evaluation.", + "assertions": [ + "Says a model should not be chosen using test-set performance", + "Points to best_trainer.json and the validation tuning metric as the selection evidence", + "Does not override Kumo's selected trial with its own ranking" + ], + "expected_skill": "kumo-training-diagnostics", + "expected_script": null + }, + { + "id": "kdiag-04", + "prompt": "What's the job ID of my Kumo training run, and can you poll its status until it's done?", + "expected_output": "Kumo trains synchronously in the notebook process: there is no hosted job ID, status API or polling. A progress file can show recorded elapsed and estimated time, but a stale file is not proof the process is running or dead.", + "assertions": [ + "Says Kumo runs synchronously in the notebook process, with no job ID or status API", + "Does not invent a job ID, status call or polling loop" + ], + "expected_skill": "kumo-training-diagnostics", + "expected_script": null + }, + { + "id": "kdiag-05", + "prompt": "Plot the ROC curve and confusion matrix for my scikit-learn LogisticRegression on X_test, y_test.", + "expected_output": "A scikit-learn evaluation task unrelated to Kumo. The agent answers with sklearn.metrics, without loading the Kumo training-diagnostics skill.", + "assertions": [ + "Does not load or follow the kumo-training-diagnostics skill", + "Uses scikit-learn metrics to plot the ROC curve and confusion matrix" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/agent-skills/nvskills/kumo-training-diagnostics/skill-card.md b/agent-skills/nvskills/kumo-training-diagnostics/skill-card.md new file mode 100644 index 000000000..d5e29a9e3 --- /dev/null +++ b/agent-skills/nvskills/kumo-training-diagnostics/skill-card.md @@ -0,0 +1,45 @@ +## Description:
+Evaluate an existing Kumo training run in a Python notebook using aggregate metrics, trial results, learning curves, refit evidence and training progress.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to answer how a Kumo model performed, which trial was selected and what its learning curves and progress show, without retraining or tuning.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/diagnostics.md](references/diagnostics.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown assessment with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Inspection only; does not tune, rerun or deploy a model.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/agent-skills/nvskills/kumoai-reference/evals/evals.json b/agent-skills/nvskills/kumoai-reference/evals/evals.json new file mode 100644 index 000000000..1272a510a --- /dev/null +++ b/agent-skills/nvskills/kumoai-reference/evals/evals.json @@ -0,0 +1,77 @@ +{ + "skill_name": "kumoai-reference", + "evals": [ + { + "id": "kref-01", + "prompt": "This request has passed kumo-data-science intake and both gates. Confirmed: entity is each account (accounts.account_id); outcome is at least one transaction with transactions.is_fraud = 1; window is the next 30 days; transactions links to accounts by account_id. Write the Kumo predictive query (PQL) for it.", + "expected_output": "`PREDICT COUNT(transactions.* WHERE transactions.is_fraud = 1, 0, 30, days) > 0 FOR EACH accounts.account_id`. The fraud condition is inline in the aggregation because it describes the future event; a trailing WHERE would instead filter which accounts are eligible and predict any transaction. The window is in the aggregation arguments with the plural unit `days`.", + "assertions": [ + "Writes PREDICT ... FOR EACH accounts.account_id", + "Puts the is_fraud = 1 condition inside the aggregation (inline WHERE), not as a trailing entity WHERE", + "Expresses the horizon as aggregation arguments 0, 30, days", + "Uses a comparison such as > 0 so the target is binary classification", + "Does not use keywords such as WITHIN or a singular unit such as day" + ], + "expected_skill": "kumoai-reference", + "expected_script": null + }, + { + "id": "kref-02", + "prompt": "Validated and confirmed by kumo-data-science: Parquet files in /data/shop (customers, orders). customers: primary key customer_id. orders: primary key order_id, time column ordered_at, link orders.customer_id -> customers.customer_id. Query: PREDICT COUNT(orders.*, 0, 30, days) = 0 FOR EACH customers.customer_id. Prediction time 2026-11-01. Threshold 0.5 [default]. Destination: the empty directory /data/scores/. Generate the Python script with the Kumo SDK.", + "expected_output": "A script that imports `kumo.engine`, connects with `FileConnector(root_dir='/data/shop')`, builds Tables with `Table.from_source_table`, links them with an `Edge`, validates the graph and the query (and checks `get_task_type()`), calls `graph.snapshot()`, generates the training and prediction tables (prediction_time a naive datetime), fits exactly once with `trainer.fit(training_table, snapshot=snapshot)`, and predicts with `trainer.predict(prediction_table, snapshot=snapshot, binary_classification_threshold=0.5, output_config=...)` writing to /data/scores/. No credentials, no environment setup, no superseded API.", + "assertions": [ + "Imports kumo.engine (not kumoai)", + "Connects with FileConnector rooted at /data/shop", + "Declares the orders -> customers relationship with an Edge object, not a dict", + "Calls graph.snapshot() and passes snapshot= to both fit and predict", + "Calls fit exactly once, on a training table generated from the predictive query", + "Passes binary_classification_threshold=0.5 to predict, not to PredictionOutputConfig", + "Uses a naive datetime (no timezone) for the 2026-11-01 prediction time", + "Writes predictions to /data/scores/ through a PredictionOutputConfig", + "Does not call set_backend, LocalBackend, kumo.init or OutputConfig" + ], + "expected_skill": "kumoai-reference", + "expected_script": null + }, + { + "id": "kref-03", + "prompt": "My old Kumo notebook fails with `AttributeError: module 'kumo' has no attribute 'set_backend'`. It does:\n\n```python\nkumo.set_backend(kumo.LocalBackend())\ngraph = kumo.Graph(tables=tables, edges=[{'src_table': 'orders', 'fkey': 'customer_id', 'dst_table': 'customers'}])\ntrainer.fit(graph, pquery=pq)\nout = OutputConfig(table_name='churn')\n```\n\nCan you patch it so it runs on the current Kumo SDK?", + "expected_output": "It says the notebook targets a superseded build of the Kumo SDK and must be rewritten, not patched line by line: there is no backend to set; edges are `Edge` objects; training is `fit(training_table, snapshot=snapshot)` after `graph.snapshot()` and table generation; the output config is `kumo.PredictionOutputConfig`. It does not keep `set_backend`, dict edges or `OutputConfig`.", + "assertions": [ + "Says the code targets a superseded (old) Kumo SDK build", + "Removes set_backend / LocalBackend rather than finding a replacement call for it", + "Rewrites the edge as an Edge object", + "Rewrites training as fit(training_table, snapshot=snapshot) with an explicit graph.snapshot()", + "Replaces OutputConfig with kumo.PredictionOutputConfig" + ], + "expected_skill": "kumoai-reference", + "expected_script": null + }, + { + "id": "kref-04", + "prompt": "Validated request: forecast weekly sales amount for each store for the next 4 weeks, from sales(store_id, sold_at, amount) linked to stores(store_id). Write the PQL and the Kumo SDK steps for the training and prediction tables.", + "expected_output": "Forecasting is a temporal regression query plus a generation plan: `PREDICT SUM(sales.amount, 0, 7, days) FOR EACH stores.store_id`, with `TrainingTableGenerationPlan(forecast_length=4)` passed to `generate_training_table(snapshot, plan=...)`, and the fitted `trainer.training_table_generation_plan` passed to `generate_prediction_table`. `forecast_length` is not a model-plan setting, and `weeks` is not a valid unit.", + "assertions": [ + "Writes a SUM aggregation over a 7-day window, e.g. PREDICT SUM(sales.amount, 0, 7, days) FOR EACH stores.store_id", + "Does not use weeks as the unit", + "Sets forecast_length=4 on a TrainingTableGenerationPlan passed to generate_training_table", + "Does not put forecast_length on the model plan", + "Reuses trainer.training_table_generation_plan when generating the prediction table" + ], + "expected_skill": "kumoai-reference", + "expected_script": null + }, + { + "id": "kref-05", + "prompt": "Write a PyTorch Geometric GCN from scratch for node classification on the Cora dataset.", + "expected_output": "A generic graph neural network task, unrelated to the Kumo SDK. The agent writes PyTorch Geometric code without loading the Kumo reference skill.", + "assertions": [ + "Does not load or follow the kumoai-reference skill", + "Writes PyTorch Geometric code for a GCN on Cora", + "Does not use kumo.engine" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/agent-skills/nvskills/kumoai-reference/skill-card.md b/agent-skills/nvskills/kumoai-reference/skill-card.md new file mode 100644 index 000000000..438ebb66b --- /dev/null +++ b/agent-skills/nvskills/kumoai-reference/skill-card.md @@ -0,0 +1,50 @@ +## Description:
+Reference for translating a validated ML request into KumoAI graph configuration, PQL (predictive query language) and Kumo SDK code.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill, normally through the kumo-data-science skill, so a coding agent writes Kumo graph definitions, predictive queries and training and prediction code that match the shipped Kumo SDK.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Optional]
+**Credential Type(s):** [Cloud Credentials, OAuth Token, Other [Snowflake connections.toml entry]]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/graph_construction.md](references/graph_construction.md)
+- [references/pql_writing.md](references/pql_writing.md)
+- [references/kumo_finetuning_sdk.md](references/kumo_finetuning_sdk.md)
+- [references/task_journeys.md](references/task_journeys.md)
+- [references/outputs.md](references/outputs.md)
+- [references/troubleshooting_signals.md](references/troubleshooting_signals.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python and PQL code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Generates code only; does not run jobs or interpret metrics.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/agent-skills/tests/test_no_environment_setup.py b/agent-skills/tests/test_no_environment_setup.py index 5f1c1bb6d..15637274d 100644 --- a/agent-skills/tests/test_no_environment_setup.py +++ b/agent-skills/tests/test_no_environment_setup.py @@ -7,14 +7,17 @@ package-file path, and the agent asked the user for one anyway: the placeholder existed only inside the rule that forbade it. So the words are checked by absence, in every file under `skills/`, including the host files about Snowflake ML Jobs, where Kumo reaches the job through host-supplied -values (`KUMO_JOB_RUNTIME`, `KUMO_JOB_REQUIREMENTS`) that the skill only names. +values (`KUMO_JOB_RUNTIME`, `KUMO_JOB_REQUIREMENTS`) that the skill only names. The catalog's skill +cards and evals under `nvskills/` are published beside the skills, so they are checked too. """ import re from pathlib import Path import pytest -SKILLS = Path(__file__).resolve().parents[1] / 'skills' +ROOT = Path(__file__).resolve().parents[1] +SKILLS = ROOT / 'skills' +CATALOG = ROOT / 'nvskills' # `pip` as a word or as the `pip_` prefix of a submission keyword; `.zip` / `.whl` as extensions, # so `zipfile.ZipFile` in the retention script does not count. @@ -22,16 +25,17 @@ def skill_files() -> list[Path]: - return sorted(p for p in SKILLS.rglob('*') + return sorted(p for root in (SKILLS, CATALOG) for p in root.rglob('*') if p.is_file() and '__pycache__' not in p.parts) def test_files_are_found() -> None: names = {p.name for p in skill_files()} - assert {'SKILL.md', 'adapter.md', 'ml_job.md', 'retain_artifacts.py'} <= names + assert {'SKILL.md', 'adapter.md', 'ml_job.md', 'retain_artifacts.py', + 'skill-card.md', 'evals.json'} <= names -@pytest.mark.parametrize('path', skill_files(), ids=lambda p: str(p.relative_to(SKILLS))) +@pytest.mark.parametrize('path', skill_files(), ids=lambda p: str(p.relative_to(ROOT))) def test_no_setup_words(path: Path) -> None: hits = [f'{n}: {line.strip()}' for n, line in enumerate(path.read_text().splitlines(), 1) if SETUP_WORDS.search(line)] diff --git a/skills/kumo-custom-populations/evals/evals.json b/skills/kumo-custom-populations/evals/evals.json new file mode 100644 index 000000000..45adaf00d --- /dev/null +++ b/skills/kumo-custom-populations/evals/evals.json @@ -0,0 +1,72 @@ +{ + "skill_name": "kumo-custom-populations", + "evals": [ + { + "id": "kpop-01", + "prompt": "Train my confirmed Kumo churn query on our own labels instead of generated ones. The Snowflake table ML.LABELS.CURATED_LABELS has ENTITY1, TARGET1, TIMESTAMP, WEIGHT and SPLIT (0 train, 1 validation, 2 test). `pq` and `snapshot` are already built in the notebook.", + "expected_output": "Before any code it replies with the plan and stops for confirmation: the table and its fields, WEIGHT and SPLIT as the weight and split columns, what it validated, and the constructor `kumo.TrainingTable.from_source_table(population_connector['CURATED_LABELS'], TrainingTableSpec(weight_col='WEIGHT', split_col='SPLIT'), pq=pq, snapshot=snapshot)`, reusing the confirmed pq and snapshot, with `suggest_model_plan(..., train_table_spec=spec)`.", + "assertions": [ + "Names kumo.TrainingTable.from_source_table as the constructor", + "Uses TrainingTableSpec with weight_col='WEIGHT' and split_col='SPLIT'", + "Reuses the existing pq and snapshot", + "Passes the same spec to suggest_model_plan as train_table_spec", + "Presents the plan and stops for the user's confirmation before training" + ], + "expected_skill": "kumo-custom-populations", + "expected_script": null + }, + { + "id": "kpop-02", + "prompt": "Our Kumo labels table ML.LABELS.CURATED_LABELS has a SPLIT_LABEL column with the values 'train', 'val' and 'test'. Use it as the split column and train.", + "expected_output": "Reports SPLIT_LABEL as the offending field: the split column reads only 0, 1 and 2, so string labels match no split and their rows are skipped (the build fails with 'custom-TT source kept no rows' if none remain). It shows the labels and their counts from a read-only query and asks for either a prepared table with 0/1/2 or explicit approval of a shown mapping, which would then be applied in a read-only connector.sql(...) source. It writes nothing back.", + "assertions": [ + "Says the split column must hold 0, 1 or 2 and that 'train'/'val'/'test' will not match", + "Explains that unmatched rows are skipped, so training could keep no rows", + "Asks for a prepared 0/1/2 table or explicit approval of a mapping it shows", + "Proposes the mapping only as a read-only connector.sql(...) source, after approval", + "Does not UPDATE the table, CREATE TABLE AS, or write a derived table" + ], + "expected_skill": "kumo-custom-populations", + "expected_script": null + }, + { + "id": "kpop-03", + "prompt": "For my Kumo custom training table, just normalize the WEIGHT column to mean 1 and multiply positives by 10. You have my approval, go ahead.", + "expected_output": "An approval given before the formula and distribution are shown is not approval. It states the exact formula and the columns it reads, shows the resulting weight distribution (min, mean, max, counts per label) from a read-only query, and stops for explicit approval. Only then would it apply the formula in a read-only connector.sql(...) source; it never writes to the source table.", + "assertions": [ + "Does not apply the weight change straight away despite the up-front approval", + "Shows the exact weight formula and the columns it reads", + "Shows or proposes a read-only query for the resulting weight distribution", + "Stops for explicit approval after showing them", + "Does not UPDATE the source table or write a derived table" + ], + "expected_skill": "kumo-custom-populations", + "expected_script": null + }, + { + "id": "kpop-04", + "prompt": "Score exactly the accounts in ML.EVAL.EVAL_ACCOUNTS (ACCOUNT_ID, ANCHOR_TS, CHURNED) with my fitted Kumo trainer, so I can compare against CHURNED afterwards.", + "expected_output": "The scoring population must not carry the target, so the table cannot be passed as is. Evaluation labels (CHURNED) stay separate and are joined to the scores afterwards. It proposes, for approval, a read-only connector.sql(...) source keeping only the entity and TIMESTAMP columns under their contract names, or asks for a table without the target, and scores with `kumo.PredictionTable.from_source_table(..., pq=pq, snapshot=snapshot, plan=trainer.training_table_generation_plan)` then `trainer.predict(population, snapshot=snapshot, ...)`, comparing output entity and anchor coverage with the population.", + "assertions": [ + "Says the scoring population must not contain the target column CHURNED", + "Keeps CHURNED separate for joining to the scores afterwards", + "Proposes a read-only connector.sql(...) source with the entity and timestamp columns, or asks for a table without the target", + "Uses kumo.PredictionTable.from_source_table with pq and snapshot", + "Says output rows or coverage will be compared with the population rather than assumed equal" + ], + "expected_skill": "kumo-custom-populations", + "expected_script": null + }, + { + "id": "kpop-05", + "prompt": "Split my pandas DataFrame into stratified train, validation and test sets (70/15/15) with scikit-learn.", + "expected_output": "A scikit-learn data-splitting task unrelated to Kumo. The agent answers with train_test_split, without loading the Kumo custom-populations skill.", + "assertions": [ + "Does not load or follow the kumo-custom-populations skill", + "Uses scikit-learn train_test_split with stratification to produce 70/15/15 splits" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/skills/kumo-custom-populations/skill-card.md b/skills/kumo-custom-populations/skill-card.md new file mode 100644 index 000000000..8a1a901ae --- /dev/null +++ b/skills/kumo-custom-populations/skill-card.md @@ -0,0 +1,45 @@ +## Description:
+Train with caller-supplied labels, weights and splits or score exact entity/anchor rows through Snowflake source tables using the local Kumo API.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents when generated Kumo training or prediction tables do not represent the rows they need, for example curated labels, manual splits, sample weights or an exact scoring population.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Yes]
+**Credential Type(s):** [Other [Snowflake connection or Snowpark session of the user's notebook]]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/populations.md](references/populations.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python and SQL code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Reads source tables only through read-only queries; never writes back to them.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index 0d491be97..d03765cca 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-17-2ab10b0f-dirty`** +**`SKILL-BUILD: 2026-11-17-1dc2be3f`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/evals/evals.json b/skills/kumo-data-science/evals/evals.json new file mode 100644 index 000000000..a82cdf78a --- /dev/null +++ b/skills/kumo-data-science/evals/evals.json @@ -0,0 +1,116 @@ +{ + "skill_name": "kumo-data-science", + "evals": [ + { + "id": "kds-01", + "prompt": "Use my sales data to figure out which customers are risky. Build me a Kumo model for it.", + "expected_output": "Treats this as a `train` request with required fields missing. In one ask-back it asks for where the data lives (for example a database and schema, or a folder of files), the observable outcome that makes a customer 'risky' (the positive-label condition), and the prediction window. It does not ask for columns, keys, joins or an output table, does not guess an outcome, and generates no code.", + "assertions": [ + "Asks where the data lives, as a location such as a database and schema, a dataset, a bucket prefix or a folder of files", + "Asks for a concrete, observable outcome that defines 'risky' (the positive-label condition) instead of inventing one", + "Asks for the prediction window (for example 'in the next 30 days')", + "Asks for all missing fields together in a single reply", + "Does not ask for column names, primary keys, joins or an output table", + "Does not write a notebook, Kumo SDK code or a predictive query" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-02", + "prompt": "Create a notebook that trains a Kumo model for our fraud use case. The data is in Snowflake, USER_DATA.E_COMMERCE. Entity: each customer. Outcome: at least one fraud event. Window: next 30 days. You can't reach our Snowflake account from here, so this is the structure; no keys are declared:\n\nCUSTOMERS(CUSTOMER_ID NUMBER, COUNTRY VARCHAR, SIGNUP_AT TIMESTAMP_NTZ)\nFRAUD_EVENTS(EVENT_ID NUMBER, CUSTOMER_ID NUMBER, EVENT_AT TIMESTAMP_NTZ, AMOUNT NUMBER, RESOLVED_AT TIMESTAMP_NTZ, DAYS_TO_RESOLVE NUMBER)\nTRANSACTIONS(TRANSACTION_ID NUMBER, CUSTOMER_ID NUMBER, CREATED_AT TIMESTAMP_NTZ, AMOUNT NUMBER)", + "expected_output": "All required fields are present, so it builds the graph proposal from the pasted structure and stops for confirmation. The Source scope line says the structure came from the user. Keys CUSTOMERS.CUSTOMER_ID, FRAUD_EVENTS.EVENT_ID and TRANSACTIONS.TRANSACTION_ID are proposed by name and, since no probe can run, tagged as unverified; time columns are EVENT_AT and CREATED_AT; FRAUD_EVENTS.CUSTOMER_ID and TRANSACTIONS.CUSTOMER_ID link to CUSTOMERS.CUSTOMER_ID; table mode all-connected [default]. An Excluded columns section removes FRAUD_EVENTS.RESOLVED_AT and DAYS_TO_RESOLVE as not knowable at the row's EVENT_AT. It ends by asking the user to confirm or correct any line, with no Kumo code.", + "assertions": [ + "Presents a graph proposal (metadata, links, table mode) built from the pasted structure rather than asking for the structure again", + "Tags proposal values with provenance tags such as [supplied], [inferred], [inferred, unverified] or [default]", + "Says the structure came from the user rather than from a read of the source", + "Proposes links from FRAUD_EVENTS.CUSTOMER_ID and TRANSACTIONS.CUSTOMER_ID to CUSTOMERS.CUSTOMER_ID", + "Lists FRAUD_EVENTS.RESOLVED_AT and FRAUD_EVENTS.DAYS_TO_RESOLVE as excluded columns because their values are not known at the event time", + "Asks the user to confirm or correct the proposal and stops there", + "Does not emit Kumo SDK code or a predictive query in this reply" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-03", + "prompt": "Build a Kumo notebook that predicts, for each account in our BigQuery dataset acme-prod:billing, whether the account will have at least one failed payment in the next 14 days.", + "expected_output": "The request has every required field. Before preflight it checks whether it can read the dataset's columns in this session (a BigQuery tool, an authenticated `bq` on the PATH, or Python with Kumo and ambient credentials). It cannot, so it sends one ask-back saying what it checked and what happened, and offers two ways forward: the user makes a read tool available, or pastes the structure (tables, columns, types, declared keys). It never asks for a credential value, does not sign in or change tool configuration, does not invent tables, and writes no code.", + "assertions": [ + "Says what it checked to read the dataset and what the result was", + "Offers the user a way to grant read access (an authenticated tool or a connected tool for BigQuery)", + "Offers the alternative of pasting the structure: tables, columns with types and any declared keys", + "Does not ask for a password, key, token or other credential value in the chat", + "Does not invent tables, columns or a graph proposal for acme-prod:billing", + "Does not re-ask for the entity, outcome or window, which the user already gave", + "Does not generate the notebook in this reply" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-04", + "prompt": "Earlier in this session you proposed this graph for our fraud request (Snowflake USER_DATA.E_COMMERCE; entity: each customer; outcome: at least one fraud event; window: next 30 days):\n\nMetadata\n- CUSTOMERS: primary key CUSTOMER_ID [inferred]\n- FRAUD_EVENTS: primary key EVENT_ID [inferred]; time column EVENT_AT [inferred]\nLinks\n- FRAUD_EVENTS.CUSTOMER_ID -> CUSTOMERS.CUSTOMER_ID [inferred]\nExcluded columns\n- FRAUD_EVENTS.RESOLVED_AT [inferred]\n\nI confirm the graph. Please write the predictions to ANALYTICS.ML.fraud_scores.", + "expected_output": "With the graph confirmed it moves to the task interpretation gate and restates the task in the documented shape: one prediction per CUSTOMERS.CUSTOMER_ID; outcome at least one fraud event; window 0 to 30 days after the prediction anchor; temporal binary classification; predictions written to ANALYTICS.ML.fraud_scores, verbatim, in the same notebook; threshold 0.5 [default]. It says whether the destination already exists and would be replaced, or that it could not check. It then asks for explicit confirmation and emits no Kumo code.", + "assertions": [ + "Restates the entity as one prediction per customer (CUSTOMERS.CUSTOMER_ID)", + "Restates the outcome as at least one fraud event and the window as the next 30 days", + "Names the task type as temporal binary classification", + "Carries the destination ANALYTICS.ML.fraud_scores verbatim, without changing its case or adding a suffix", + "States a classification threshold of 0.5 as the default", + "Says whether the destination table already exists and would be replaced on every run, or that it could not check", + "Asks for explicit confirmation and does not emit Kumo SDK code or a notebook yet" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-05", + "prompt": "I trained a Kumo model in my notebook an hour ago and `training_result` is still in memory. Which columns influenced its predictions the most?", + "expected_output": "This is an attribution question about an existing run, not a new training request. It routes to the kumo-model-explanations skill (or says that skill is not available here), keeps the existing run, and does not run new-training intake or start another fit.", + "assertions": [ + "Treats the request as an explanation question about the existing run, handled by the kumo-model-explanations skill (or reports that skill is unavailable rather than improvising its workflow)", + "Does not ask for the data location, entity, outcome or prediction window", + "Does not start, propose or generate a new training run" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-06", + "prompt": "Schedule the Kumo churn notebook you generated to run every night at 2am as a job on our cluster, and tell me when the first run finishes.", + "expected_output": "In this generic coding-agent host there is no job file: the skill does not submit, schedule or monitor jobs. It says so and offers the notebook or script to run themselves. It does not create a scheduler entry or submit anything.", + "assertions": [ + "Says that in this environment the skill does not submit, schedule or monitor jobs", + "Offers the notebook or a script as what it can provide instead", + "Does not create a cron entry, scheduler configuration or job submission", + "Does not claim to watch or report on a run" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-07", + "prompt": "Set up kumo.init with my KUMO_API_KEY and train a churn model on Kumo's hosted service.", + "expected_output": "Hosted Kumo (kumo.init, API keys, KUMO_API_KEY / KUMO_API_ENDPOINT, Kumo Studio) is not the SDK this skill targets. It says so and stops, and does not blend hosted setup into generated code.", + "assertions": [ + "Says that hosted Kumo (kumo.init with an API key) is not the SDK this skill works with", + "Does not write code that calls kumo.init or reads KUMO_API_KEY", + "Does not ask the user to paste an API key" + ], + "expected_skill": "kumo-data-science", + "expected_script": null + }, + { + "id": "kds-08", + "prompt": "Write a pandas function that removes duplicate rows from a DataFrame, keeping the most recent row per order_id by updated_at.", + "expected_output": "A plain pandas task with no Kumo or predictive-model element. The agent answers directly, for example with sort_values plus drop_duplicates, without loading the Kumo skill or starting its intake.", + "assertions": [ + "Does not load or follow the kumo-data-science skill or ask Kumo intake questions", + "Provides pandas code that keeps the latest row per order_id by updated_at" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/skills/kumo-data-science/skill-card.md b/skills/kumo-data-science/skill-card.md new file mode 100644 index 000000000..9d4179590 --- /dev/null +++ b/skills/kumo-data-science/skill-card.md @@ -0,0 +1,50 @@ +## Description:
+Turns a plain-language request for a KumoAI predictive model on relational data into a confirmed task and graph through a multi-turn intake, generates one notebook or script that trains and predicts with the local Kumo SDK, and routes inspection, persistence, population and tuning requests to their own Kumo skills.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to build Kumo models for churn, propensity, fraud, recommendation, forecasting and link prediction on data in Snowflake, BigQuery, Databricks, S3, GCS or local files: the agent checks the request, proposes the graph for confirmation and writes the training-and-prediction notebook.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Optional]
+**Credential Type(s):** [Cloud Credentials, OAuth Token, Other [Snowflake connections.toml entry]]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [accept/intake_contract.md](accept/intake_contract.md)
+- [accept/preflight.md](accept/preflight.md)
+- [accept/batch_inference.md](accept/batch_inference.md)
+- [validate/atomic_question_check.md](validate/atomic_question_check.md)
+- [hosts/generic/adapter.md](hosts/generic/adapter.md)
+- [hosts/generic/runtime.md](hosts/generic/runtime.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown replies (ask-backs, graph proposal, task restatement) and one Jupyter notebook or Python script]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Generates code only; runs it only when the user asks. Writes predictions only to a destination the user confirmed.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/kumo-graph-visualization/evals/evals.json b/skills/kumo-graph-visualization/evals/evals.json new file mode 100644 index 000000000..0be558c67 --- /dev/null +++ b/skills/kumo-graph-visualization/evals/evals.json @@ -0,0 +1,67 @@ +{ + "skill_name": "kumo-graph-visualization", + "evals": [ + { + "id": "kviz-01", + "prompt": "Add a graph diagram step to my Kumo training script churn_30d_train.py.", + "expected_output": "Adds, after `graph.validate()` and before snapshot and training, a step that saves the diagram next to the script as `churn_30d_train.graph.svg` via `Path(__file__).with_suffix('.graph.svg')`, guarded by `hasattr(graph, 'visualize')` and a try/except that prints the error and lets training continue, and prints the saved path.", + "assertions": [ + "Saves the diagram with graph.visualize to a path derived from the script name, churn_30d_train.graph.svg", + "Places the step after graph validation and before snapshot creation or training", + "Guards with hasattr(graph, 'visualize') and a try/except that lets the script continue", + "Prints or reports the saved diagram path" + ], + "expected_skill": "kumo-graph-visualization", + "expected_script": null + }, + { + "id": "kviz-02", + "prompt": "Export my Kumo graph to reports/graph.pdf. That file already exists from last week.", + "expected_output": "A view/export request needs no training intake. `.pdf` gives a diagram plus a complete schema index. Because the named file exists, it says so and asks for confirmation before replacing it, then uses `graph.visualize('reports/graph.pdf')`. Samples stay off.", + "assertions": [ + "Does not ask model-training intake questions", + "Uses graph.visualize with the .pdf path", + "Says the file already exists and asks before replacing it", + "Does not turn on show_samples" + ], + "expected_skill": "kumo-graph-visualization", + "expected_script": null + }, + { + "id": "kviz-03", + "prompt": "I'm deciding which table to add to my Kumo graph next. Give me a text view of all its columns, keys and relationships that you can read.", + "expected_output": "Exports the graph to Markdown in a new temporary directory (`graph.visualize(schema_path)` with a `.md` path), reads that file for columns, key and time roles and relationships, and notes the schema does not measure predictive usefulness.", + "assertions": [ + "Exports the graph to a .md file with graph.visualize", + "Writes it to a new temporary directory so nothing is replaced", + "Reads the exported file to inspect columns, keys and relationships", + "Says the schema view does not measure how useful a table would be for prediction" + ], + "expected_skill": "kumo-graph-visualization", + "expected_script": null + }, + { + "id": "kviz-04", + "prompt": "Show my Kumo graph in the notebook. It has 45 tables, and employees.manager_id links back to employees.employee_id.", + "expected_output": "The viewer supports up to 30 tables and does not support self-relations, so this graph cannot be fully shown. It reports that and lets modeling continue with the original graph, without changing the graph to make it render.", + "assertions": [ + "Says the viewer supports at most 30 tables", + "Says self-relations such as employees.manager_id -> employees are not supported", + "Does not remove tables or relationships from the model's graph to make it render" + ], + "expected_skill": "kumo-graph-visualization", + "expected_script": null + }, + { + "id": "kviz-05", + "prompt": "Draw my social network from edges.csv (source, target) with networkx and matplotlib.", + "expected_output": "A networkx plotting task unrelated to Kumo. The agent answers with networkx code without loading the Kumo graph-visualization skill.", + "assertions": [ + "Does not load or follow the kumo-graph-visualization skill", + "Uses networkx to read edges.csv and matplotlib to draw it" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/skills/kumo-graph-visualization/skill-card.md b/skills/kumo-graph-visualization/skill-card.md new file mode 100644 index 000000000..9eed168ad --- /dev/null +++ b/skills/kumo-graph-visualization/skill-card.md @@ -0,0 +1,44 @@ +## Description:
+Provide graph visuals in Kumo notebooks and SDK scripts, display or export relationships on request, and inspect Markdown schemas during model iteration.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to view a Kumo graph's tables and relationships in a notebook, export it as SVG, PDF or Markdown, and read its schema while deciding on graph changes.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks; SVG, PDF or Markdown graph files when exported]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Source sample values are off unless the user asks for them.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/kumo-model-explanations/evals/evals.json b/skills/kumo-model-explanations/evals/evals.json new file mode 100644 index 000000000..08061a6c7 --- /dev/null +++ b/skills/kumo-model-explanations/evals/evals.json @@ -0,0 +1,81 @@ +{ + "skill_name": "kumo-model-explanations", + "evals": [ + { + "id": "kexpl-01", + "prompt": "Which columns influenced my Kumo model the most? I fit it in this notebook and `training_result` is still in memory.", + "expected_output": "Uses `training_result.explanations`, checks `availability['gradient']`, and reads `gradient_scores().to_pandas()` (optionally `plot_gradient`). Names the result Kumo gradient attribution over the processed TEST subset, normalized across the graph, and not drivers, probabilities, percentages or causal effects. Reports the run, stage and file paths, and notes the files are local scratch excluded from the published model bundle. Does not retrain.", + "assertions": [ + "Uses training_result.explanations and checks availability before reading", + "Reads gradient scores with gradient_scores() or plot_gradient()", + "Calls the scores Kumo gradient attribution over the processed test subset", + "Says the scores are not causal effects, probabilities or percentages", + "Reports the run and the TEST stage the scores come from", + "Does not start another fit" + ], + "expected_skill": "kumo-model-explanations", + "expected_script": null + }, + { + "id": "kexpl-02", + "prompt": "For my Kumo model, compare predictions with actual outcomes across age cohorts. `training_result` is in memory.", + "expected_output": "Uses column analysis: `explanations.column_analysis()`, `cohort_keys()` to find the (table, hop, column) key, then `cohort(key)` / `plot_cohort(key)`. Reports each cohort's population share beside its mean prediction and, when recorded, mean target. A cohort mismatch is a lead to investigate, not proof of leakage or causation; prediction_score and target_score are not AUROC or error metrics.", + "assertions": [ + "Uses column analysis with cohort_keys() and cohort() or plot_cohort()", + "Reports population share alongside mean prediction and mean target per cohort", + "Says a prediction-versus-target mismatch is a lead, not proof of leakage or a cause", + "Does not treat prediction_score or target_score as AUROC or an error metric" + ], + "expected_skill": "kumo-model-explanations", + "expected_script": null + }, + { + "id": "kexpl-03", + "prompt": "I reloaded my Kumo model with `Trainer.load(bundle_dir)` and there are no explanations. Just call `trainer.explain()` to regenerate them.", + "expected_output": "There is no public `Trainer.explain()` or regeneration API. Published bundles exclude explanation files, so a loaded bundle reports them unavailable (`model_only`). The retained training run can be reopened with `kumo.TrainingExplanations.load(run_dir)`. It says no fit was started and asks whether the user wants one.", + "assertions": [ + "Says there is no public Trainer.explain() to regenerate explanations", + "Says the model bundle does not contain explanation files", + "Suggests reopening the retained run with kumo.TrainingExplanations.load(run_dir)", + "Does not start a fit, and asks before proposing one" + ], + "expected_skill": "kumo-model-explanations", + "expected_script": null + }, + { + "id": "kexpl-04", + "prompt": "Run column analysis and radiant attribution on my Kumo link-prediction model.", + "expected_output": "Kumo calls the method gradient attribution, not Radiant. Column analysis is restricted to supported node-level tasks and is not available for link prediction. Gradient scoring may exist, but TWO_TOWER link models are skipped, so availability must be checked.", + "assertions": [ + "Says Kumo calls the method gradient attribution", + "Says column analysis is not available for link prediction", + "Says gradient attribution availability must be checked for this link model (TWO_TOWER models are skipped)" + ], + "expected_skill": "kumo-model-explanations", + "expected_script": null + }, + { + "id": "kexpl-05", + "prompt": "There are three Kumo training runs under /scratch/kumo/runs. Show me the gradient attribution.", + "expected_output": "Lists the candidate runs with their paths and asks which one the user means before showing any score. It never picks the newest directory or combines runs.", + "assertions": [ + "Lists the candidate runs with their paths", + "Asks which run the user means before showing scores", + "Does not pick the newest run or combine runs" + ], + "expected_skill": "kumo-model-explanations", + "expected_script": null + }, + { + "id": "kexpl-06", + "prompt": "Compute SHAP values for my XGBoost classifier and show a beeswarm plot.", + "expected_output": "A SHAP task on an XGBoost model, unrelated to Kumo. The agent uses the shap library without loading the Kumo model-explanations skill.", + "assertions": [ + "Does not load or follow the kumo-model-explanations skill", + "Uses the shap library to compute values and draw a beeswarm plot" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/skills/kumo-model-explanations/skill-card.md b/skills/kumo-model-explanations/skill-card.md new file mode 100644 index 000000000..e231160f8 --- /dev/null +++ b/skills/kumo-model-explanations/skill-card.md @@ -0,0 +1,45 @@ +## Description:
+Inspect and interpret gradient attribution, column analysis and sampled explanation subgraphs from a Kumo model trained in a Python notebook.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to read the explanation artifacts of an existing Kumo run, such as which table columns the model relied on and how predictions compare with outcomes across column cohorts, without retraining.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/artifacts.md](references/artifacts.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown assessment with inline Python code blocks and plots]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Inspection only; makes no causal claims and starts no fit.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/kumo-model-persistence/evals/evals.json b/skills/kumo-model-persistence/evals/evals.json new file mode 100644 index 000000000..6dcdc4911 --- /dev/null +++ b/skills/kumo-model-persistence/evals/evals.json @@ -0,0 +1,83 @@ +{ + "skill_name": "kumo-model-persistence", + "evals": [ + { + "id": "kpersist-01", + "prompt": "Train the confirmed Kumo churn model (model_plan, training_table and snapshot are ready in my Snowflake notebook) and publish it to the model registry in ML.REGISTRY as CHURN_MODEL version V3.", + "expected_output": "Publishes during the fit: `trainer.fit(training_table, snapshot=snapshot, export_config=kumo.ModelOutputConfig(connector=registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V3')))` with a connector scoped to ML.REGISTRY, then checks `training_result.published_model` and raises if it is None. Uses `location`, not `path`, for Snowflake. Checks that V3 does not already exist first and does not overwrite an existing version.", + "assertions": [ + "Passes a kumo.ModelOutputConfig as export_config to trainer.fit", + "Uses kumo.ModelRegistryLocation with model_name CHURN_MODEL and version V3", + "Uses location= rather than path= for the Snowflake registry", + "Checks training_result.published_model and treats None as a failure", + "Checks that version V3 does not already exist and does not overwrite an existing version" + ], + "expected_skill": "kumo-model-persistence", + "expected_script": null + }, + { + "id": "kpersist-02", + "prompt": "Load CHURN_MODEL from our Snowflake model registry in ML.REGISTRY and score new customers. SHOW VERSIONS lists V1, V2 and V3.", + "expected_output": "Several versions exist and the user named none, so it asks which version to load rather than choosing one. It explains that `version=None` loads the registry default, which is not necessarily the newest, and that 'latest' would mean the newest created_on, passed explicitly. It does not retrain.", + "assertions": [ + "Asks which version to load instead of picking one", + "Explains that loading without a version uses the registry default, not necessarily the newest", + "Does not train a new model" + ], + "expected_skill": "kumo-model-persistence", + "expected_script": null + }, + { + "id": "kpersist-03", + "prompt": "Load version V2 of CHURN_MODEL from the Snowflake registry ML.REGISTRY and score our customers as of 2026-12-01. The graph-building code and query are in our training record for V2.", + "expected_output": "Restores with `kumo.Trainer.load_from_connector(registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V2'))`, rebuilds the graph from the V2 training record (not the live schema), creates a fresh `graph.snapshot()`, generates the prediction table with a naive `prediction_time` and `plan=loaded.training_table_generation_plan`, and predicts with `snapshot=` and an explicit binary threshold. No retraining.", + "assertions": [ + "Loads with kumo.Trainer.load_from_connector and an explicit ModelRegistryLocation for version V2", + "Rebuilds the graph from the V2 training record rather than from the live schema or another version", + "Builds a new graph.snapshot() for scoring", + "Passes plan=loaded.training_table_generation_plan to generate_prediction_table", + "Passes snapshot= to predict and an explicit binary classification threshold", + "Does not call fit" + ], + "expected_skill": "kumo-model-persistence", + "expected_script": null + }, + { + "id": "kpersist-04", + "prompt": "Before this Snowflake notebook shuts down, keep my Kumo run's aggregate metrics and the per-trial validation results on the internal stage @ML.ARTIFACTS so I can look at them next week.", + "expected_output": "Selects explicit files from one run (not the whole scratch tree, which can hold source data and credentials), writes aggregate metrics to a new JSON file with allow_nan=False, builds an archive with the packaged `create_archive` helper from an explicit list of relative paths, uploads with `session.file.put(..., auto_compress=False, overwrite=False)` to a new run-specific prefix and checks exactly one `UPLOADED` result, and records the stage path and archive SHA256 digest.", + "assertions": [ + "Selects explicit files from one run instead of uploading the whole scratch directory", + "Uses the packaged create_archive helper with a list of explicit relative file paths", + "Uploads with session.file.put and overwrite=False", + "Checks that the upload status is UPLOADED and treats anything else, including SKIPPED, as not retained", + "Records the stage object path and the archive digest" + ], + "expected_skill": "kumo-model-persistence", + "expected_script": null + }, + { + "id": "kpersist-05", + "prompt": "Look up my Kumo model by its hosted job ID job-8841 and redeploy it.", + "expected_output": "The local Kumo API has no hosted model or job ID lookup. It says so and asks for the registry model name and version, or a retained bundle, instead of inventing a lookup.", + "assertions": [ + "Says there is no hosted model or job ID lookup in this local API", + "Asks for a registry model and explicit version, or a retained bundle", + "Does not invent an API call that fetches a model by job ID" + ], + "expected_skill": "kumo-model-persistence", + "expected_script": null + }, + { + "id": "kpersist-06", + "prompt": "Save my trained scikit-learn model with joblib and upload it to s3://ml-models/fraud/.", + "expected_output": "A scikit-learn persistence task unrelated to Kumo. The agent answers with joblib and an S3 upload, without loading the Kumo persistence skill.", + "assertions": [ + "Does not load or follow the kumo-model-persistence skill", + "Uses joblib to save the model and uploads it to the S3 path" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/skills/kumo-model-persistence/skill-card.md b/skills/kumo-model-persistence/skill-card.md new file mode 100644 index 000000000..1b02d2b88 --- /dev/null +++ b/skills/kumo-model-persistence/skill-card.md @@ -0,0 +1,47 @@ +## Description:
+Publish and reload Kumo models through the Snowflake registry, reconstruct compatible scoring inputs, and retain or retrieve run diagnostics and output files through a Snowflake stage.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to publish a Kumo model during its fit, reload an explicit registry version or retained bundle for scoring, and keep selected run files on a Snowflake stage after the notebook session ends.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Yes]
+**Credential Type(s):** [Other [Snowflake connection or Snowpark session of the user's notebook]]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/reuse.md](references/reuse.md)
+- [references/retention.md](references/retention.md)
+- [scripts/retain_artifacts.py](scripts/retain_artifacts.py)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks; an archive with a manifest of byte lengths and SHA256 digests when artifacts are retained]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Loading a model does not authorize training, replacing a registry version or overwriting a destination.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/kumo-model-tuning/evals/evals.json b/skills/kumo-model-tuning/evals/evals.json new file mode 100644 index 000000000..2b5419d26 --- /dev/null +++ b/skills/kumo-model-tuning/evals/evals.json @@ -0,0 +1,68 @@ +{ + "skill_name": "kumo-model-tuning", + "evals": [ + { + "id": "ktune-01", + "prompt": "My Kumo `trainer` is already fitted in this notebook with the plan I tuned last week. Before a long run on new data I want a quick smoke test: one experiment, three epochs, no refits. Show me the plan changes.", + "expected_output": "Starts from the existing plan (`trainer.model_plan`), not a fresh `suggest_model_plan`, which would reset settings the user did not mention. Applies only the requested changes: `training_job.num_experiments = 1`, `training_job.refit_trainval = False`, `training_job.refit_full = False`, `optimization.max_epochs = 3`. Says this is a limited functional check, not a production-quality setting, and presents the resolved plan without running the fit.", + "assertions": [ + "Starts from the existing plan (trainer.model_plan) rather than a new suggest_model_plan", + "Sets plan.training_job.num_experiments = 1", + "Sets plan.training_job.refit_trainval = False and plan.training_job.refit_full = False", + "Sets plan.optimization.max_epochs = 3", + "Says the smoke test is a limited functional check, not a production-quality configuration", + "Does not run fit as part of presenting the plan" + ], + "expected_skill": "kumo-model-tuning", + "expected_script": null + }, + { + "id": "ktune-02", + "prompt": "In my existing Kumo model plan, use a numerical encoder for customers.account_balance. The plan already has encoder overrides for two other columns.", + "expected_output": "Imports `Numerical` from `kumoapi.encoder` and adds `'customers.account_balance': Numerical()` to `plan.column_processing.encoder_overrides`, merging into the existing overrides rather than replacing them. Notes that the key must match the graph's actual table.column and semantic type.", + "assertions": [ + "Uses Numerical from kumoapi.encoder", + "Sets the override under plan.column_processing.encoder_overrides with the key 'customers.account_balance'", + "Merges into the existing overrides instead of replacing the whole mapping", + "Notes the key must match the graph's table and column (and its semantic type)" + ], + "expected_skill": "kumo-model-tuning", + "expected_script": null + }, + { + "id": "ktune-03", + "prompt": "Set forecast_length = 4 in my Kumo model plan so the model forecasts four periods ahead.", + "expected_output": "Explains that `forecast_length` belongs to `TrainingTableGenerationPlan`, not `ModelPlan`, together with generation, split and lag settings, so it cannot be set on the model plan. Does not invent a model-plan field for it.", + "assertions": [ + "Says forecast_length is not a ModelPlan setting", + "Says it belongs to TrainingTableGenerationPlan", + "Does not set a forecast_length field on the model plan" + ], + "expected_skill": "kumo-model-tuning", + "expected_script": null + }, + { + "id": "ktune-04", + "prompt": "I want my Kumo model tuned on a custom Python metric I wrote, `def profit_at_k(y, p): ...`. Plug it into the model plan.", + "expected_output": "Says that arbitrary Python metric callables are not supported: metric names go in `plan.training_job.metrics` and `tune_metric`, chosen from those the task supports, with the tuning direction kept explicit. It may suggest a supported metric but does not wire the function in.", + "assertions": [ + "Says a custom Python metric function cannot be used by the model plan", + "Says metrics are chosen by name in plan.training_job.metrics and tune_metric", + "Does not write code that passes the Python function to the plan" + ], + "expected_skill": "kumo-model-tuning", + "expected_script": null + }, + { + "id": "ktune-05", + "prompt": "Tune the hyperparameters of my scikit-learn RandomForestClassifier with GridSearchCV over n_estimators and max_depth.", + "expected_output": "A scikit-learn task unrelated to Kumo. The agent writes GridSearchCV code without loading the Kumo model-tuning skill.", + "assertions": [ + "Does not load or follow the kumo-model-tuning skill", + "Provides GridSearchCV code over n_estimators and max_depth" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/skills/kumo-model-tuning/skill-card.md b/skills/kumo-model-tuning/skill-card.md new file mode 100644 index 000000000..9f36b8483 --- /dev/null +++ b/skills/kumo-model-tuning/skill-card.md @@ -0,0 +1,45 @@ +## Description:
+Configure a Kumo model plan for a user-defined task, including encoders, run mode, bounded experiments, sampling, optimization, architecture, metrics and refits.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to change a Kumo model plan for a confirmed task, such as a smoke test, encoder overrides or bounded experiments, while keeping the target and evaluation protocol unchanged.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/plans.md](references/plans.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Suggesting a plan does not run its experiments.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/kumo-training-diagnostics/evals/evals.json b/skills/kumo-training-diagnostics/evals/evals.json new file mode 100644 index 000000000..641ff73e1 --- /dev/null +++ b/skills/kumo-training-diagnostics/evals/evals.json @@ -0,0 +1,66 @@ +{ + "skill_name": "kumo-training-diagnostics", + "evals": [ + { + "id": "kdiag-01", + "prompt": "My Kumo churn model just finished training in this notebook and `training_result` is in memory. How did it do?", + "expected_output": "Reads `training_result.metrics` as an attribute and lists the metric keys and finite values that are actually present, treating missing values as unavailable, not zero. For binary classification it explains that AUROC measures ranking and AP/AUPRC depends on prevalence. It does not assume metrics that were not computed and does not retrain or tune.", + "assertions": [ + "Reads training_result.metrics as an attribute, not a method call", + "Reports only the metric keys actually present, treating missing values as unavailable rather than zero", + "Explains that AUROC measures ranking quality and AP/AUPRC depends on class prevalence", + "Does not start a new fit or tuning run" + ], + "expected_skill": "kumo-training-diagnostics", + "expected_script": null + }, + { + "id": "kdiag-02", + "prompt": "Is my Kumo model overfitting? `training_result` from the fit is still in memory; look at the learning curves.", + "expected_output": "Locates the run from `training_result.bundle_dir`, finds `auto_train_progress.json` under it, and reads each trial's epoch records from `trial_progress`, keeping trials and refits separate. A widening train/validation gap can suggest overfitting, but distribution shift, split design and sampling can also explain it. It does not join unrelated trials into one curve and does not retrain.", + "assertions": [ + "Finds the run directory from training_result.bundle_dir and reads auto_train_progress.json", + "Reads per-trial epoch metrics and keeps trial and refit histories separate", + "Says a widening train/validation gap can suggest overfitting but has other possible explanations", + "Does not plot unrelated trials as one continuous curve", + "Does not start a new fit" + ], + "expected_skill": "kumo-training-diagnostics", + "expected_script": null + }, + { + "id": "kdiag-03", + "prompt": "Pick the Kumo trial with the best test AUROC from my last run and use that one instead of the one Kumo selected.", + "expected_output": "Declines to select by test-set performance. Trial selection uses the configured validation tuning metric and its direction, recorded in `best_trainer.json`; the agent should not replace the SDK's selection with its own ranking. Test results are held for final evaluation.", + "assertions": [ + "Says a model should not be chosen using test-set performance", + "Points to best_trainer.json and the validation tuning metric as the selection evidence", + "Does not override Kumo's selected trial with its own ranking" + ], + "expected_skill": "kumo-training-diagnostics", + "expected_script": null + }, + { + "id": "kdiag-04", + "prompt": "What's the job ID of my Kumo training run, and can you poll its status until it's done?", + "expected_output": "Kumo trains synchronously in the notebook process: there is no hosted job ID, status API or polling. A progress file can show recorded elapsed and estimated time, but a stale file is not proof the process is running or dead.", + "assertions": [ + "Says Kumo runs synchronously in the notebook process, with no job ID or status API", + "Does not invent a job ID, status call or polling loop" + ], + "expected_skill": "kumo-training-diagnostics", + "expected_script": null + }, + { + "id": "kdiag-05", + "prompt": "Plot the ROC curve and confusion matrix for my scikit-learn LogisticRegression on X_test, y_test.", + "expected_output": "A scikit-learn evaluation task unrelated to Kumo. The agent answers with sklearn.metrics, without loading the Kumo training-diagnostics skill.", + "assertions": [ + "Does not load or follow the kumo-training-diagnostics skill", + "Uses scikit-learn metrics to plot the ROC curve and confusion matrix" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/skills/kumo-training-diagnostics/skill-card.md b/skills/kumo-training-diagnostics/skill-card.md new file mode 100644 index 000000000..d5e29a9e3 --- /dev/null +++ b/skills/kumo-training-diagnostics/skill-card.md @@ -0,0 +1,45 @@ +## Description:
+Evaluate an existing Kumo training run in a Python notebook using aggregate metrics, trial results, learning curves, refit evidence and training progress.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to answer how a Kumo model performed, which trial was selected and what its learning curves and progress show, without retraining or tuning.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/diagnostics.md](references/diagnostics.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown assessment with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Inspection only; does not tune, rerun or deploy a model.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/kumoai-reference/evals/evals.json b/skills/kumoai-reference/evals/evals.json new file mode 100644 index 000000000..1272a510a --- /dev/null +++ b/skills/kumoai-reference/evals/evals.json @@ -0,0 +1,77 @@ +{ + "skill_name": "kumoai-reference", + "evals": [ + { + "id": "kref-01", + "prompt": "This request has passed kumo-data-science intake and both gates. Confirmed: entity is each account (accounts.account_id); outcome is at least one transaction with transactions.is_fraud = 1; window is the next 30 days; transactions links to accounts by account_id. Write the Kumo predictive query (PQL) for it.", + "expected_output": "`PREDICT COUNT(transactions.* WHERE transactions.is_fraud = 1, 0, 30, days) > 0 FOR EACH accounts.account_id`. The fraud condition is inline in the aggregation because it describes the future event; a trailing WHERE would instead filter which accounts are eligible and predict any transaction. The window is in the aggregation arguments with the plural unit `days`.", + "assertions": [ + "Writes PREDICT ... FOR EACH accounts.account_id", + "Puts the is_fraud = 1 condition inside the aggregation (inline WHERE), not as a trailing entity WHERE", + "Expresses the horizon as aggregation arguments 0, 30, days", + "Uses a comparison such as > 0 so the target is binary classification", + "Does not use keywords such as WITHIN or a singular unit such as day" + ], + "expected_skill": "kumoai-reference", + "expected_script": null + }, + { + "id": "kref-02", + "prompt": "Validated and confirmed by kumo-data-science: Parquet files in /data/shop (customers, orders). customers: primary key customer_id. orders: primary key order_id, time column ordered_at, link orders.customer_id -> customers.customer_id. Query: PREDICT COUNT(orders.*, 0, 30, days) = 0 FOR EACH customers.customer_id. Prediction time 2026-11-01. Threshold 0.5 [default]. Destination: the empty directory /data/scores/. Generate the Python script with the Kumo SDK.", + "expected_output": "A script that imports `kumo.engine`, connects with `FileConnector(root_dir='/data/shop')`, builds Tables with `Table.from_source_table`, links them with an `Edge`, validates the graph and the query (and checks `get_task_type()`), calls `graph.snapshot()`, generates the training and prediction tables (prediction_time a naive datetime), fits exactly once with `trainer.fit(training_table, snapshot=snapshot)`, and predicts with `trainer.predict(prediction_table, snapshot=snapshot, binary_classification_threshold=0.5, output_config=...)` writing to /data/scores/. No credentials, no environment setup, no superseded API.", + "assertions": [ + "Imports kumo.engine (not kumoai)", + "Connects with FileConnector rooted at /data/shop", + "Declares the orders -> customers relationship with an Edge object, not a dict", + "Calls graph.snapshot() and passes snapshot= to both fit and predict", + "Calls fit exactly once, on a training table generated from the predictive query", + "Passes binary_classification_threshold=0.5 to predict, not to PredictionOutputConfig", + "Uses a naive datetime (no timezone) for the 2026-11-01 prediction time", + "Writes predictions to /data/scores/ through a PredictionOutputConfig", + "Does not call set_backend, LocalBackend, kumo.init or OutputConfig" + ], + "expected_skill": "kumoai-reference", + "expected_script": null + }, + { + "id": "kref-03", + "prompt": "My old Kumo notebook fails with `AttributeError: module 'kumo' has no attribute 'set_backend'`. It does:\n\n```python\nkumo.set_backend(kumo.LocalBackend())\ngraph = kumo.Graph(tables=tables, edges=[{'src_table': 'orders', 'fkey': 'customer_id', 'dst_table': 'customers'}])\ntrainer.fit(graph, pquery=pq)\nout = OutputConfig(table_name='churn')\n```\n\nCan you patch it so it runs on the current Kumo SDK?", + "expected_output": "It says the notebook targets a superseded build of the Kumo SDK and must be rewritten, not patched line by line: there is no backend to set; edges are `Edge` objects; training is `fit(training_table, snapshot=snapshot)` after `graph.snapshot()` and table generation; the output config is `kumo.PredictionOutputConfig`. It does not keep `set_backend`, dict edges or `OutputConfig`.", + "assertions": [ + "Says the code targets a superseded (old) Kumo SDK build", + "Removes set_backend / LocalBackend rather than finding a replacement call for it", + "Rewrites the edge as an Edge object", + "Rewrites training as fit(training_table, snapshot=snapshot) with an explicit graph.snapshot()", + "Replaces OutputConfig with kumo.PredictionOutputConfig" + ], + "expected_skill": "kumoai-reference", + "expected_script": null + }, + { + "id": "kref-04", + "prompt": "Validated request: forecast weekly sales amount for each store for the next 4 weeks, from sales(store_id, sold_at, amount) linked to stores(store_id). Write the PQL and the Kumo SDK steps for the training and prediction tables.", + "expected_output": "Forecasting is a temporal regression query plus a generation plan: `PREDICT SUM(sales.amount, 0, 7, days) FOR EACH stores.store_id`, with `TrainingTableGenerationPlan(forecast_length=4)` passed to `generate_training_table(snapshot, plan=...)`, and the fitted `trainer.training_table_generation_plan` passed to `generate_prediction_table`. `forecast_length` is not a model-plan setting, and `weeks` is not a valid unit.", + "assertions": [ + "Writes a SUM aggregation over a 7-day window, e.g. PREDICT SUM(sales.amount, 0, 7, days) FOR EACH stores.store_id", + "Does not use weeks as the unit", + "Sets forecast_length=4 on a TrainingTableGenerationPlan passed to generate_training_table", + "Does not put forecast_length on the model plan", + "Reuses trainer.training_table_generation_plan when generating the prediction table" + ], + "expected_skill": "kumoai-reference", + "expected_script": null + }, + { + "id": "kref-05", + "prompt": "Write a PyTorch Geometric GCN from scratch for node classification on the Cora dataset.", + "expected_output": "A generic graph neural network task, unrelated to the Kumo SDK. The agent writes PyTorch Geometric code without loading the Kumo reference skill.", + "assertions": [ + "Does not load or follow the kumoai-reference skill", + "Writes PyTorch Geometric code for a GCN on Cora", + "Does not use kumo.engine" + ], + "expected_skill": null, + "expected_script": null + } + ] +} diff --git a/skills/kumoai-reference/skill-card.md b/skills/kumoai-reference/skill-card.md new file mode 100644 index 000000000..438ebb66b --- /dev/null +++ b/skills/kumoai-reference/skill-card.md @@ -0,0 +1,50 @@ +## Description:
+Reference for translating a validated ML request into KumoAI graph configuration, PQL (predictive query language) and Kumo SDK code.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill, normally through the kumo-data-science skill, so a coding agent writes Kumo graph definitions, predictive queries and training and prediction code that match the shipped Kumo SDK.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Optional]
+**Credential Type(s):** [Cloud Credentials, OAuth Token, Other [Snowflake connections.toml entry]]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/graph_construction.md](references/graph_construction.md)
+- [references/pql_writing.md](references/pql_writing.md)
+- [references/kumo_finetuning_sdk.md](references/kumo_finetuning_sdk.md)
+- [references/task_journeys.md](references/task_journeys.md)
+- [references/outputs.md](references/outputs.md)
+- [references/troubleshooting_signals.md](references/troubleshooting_signals.md)
+- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python and PQL code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Generates code only; does not run jobs or interpret metrics.]
+ +## Skill Version(s):
+Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
From 1a2852d74897bb3a84970510a4950d51a77867de Mon Sep 17 00:00:00 2001 From: igutierrez Date: Tue, 17 Nov 2026 13:11:06 -0600 Subject: [PATCH 22/35] agent-skills: cast a time column first; link skill cards to the published files Population staging parsed every text time column with pc.strptime when the entity declared a timestamp_format. That format is chrono syntax, and Arrow rejects %.f, %-m and %:z, so ISO text with fractional seconds that pc.cast used to stage (2024-01-15 10:00:00.750 with '%Y-%m-%d %H:%M:%S%.f') raised ArrowInvalid. Cast first and parse with the format only when the cast fails; a format Arrow cannot read is reported, not guessed. The skill cards' References linked to paths that exist only beside the published copy, so the repository docs check failed on their source under agent-skills/nvskills/. They now link to the published files on main. Co-Authored-By: Claude Opus 5.5 --- .../kumo-custom-populations/skill-card.md | 2 +- .../nvskills/kumo-data-science/skill-card.md | 12 ++++++------ .../kumo-model-explanations/skill-card.md | 2 +- .../kumo-model-persistence/skill-card.md | 6 +++--- .../nvskills/kumo-model-tuning/skill-card.md | 2 +- .../kumo-training-diagnostics/skill-card.md | 2 +- .../nvskills/kumoai-reference/skill-card.md | 12 ++++++------ agent-skills/skills/hosts/generic/runtime.md | 16 +++++++++++----- skills/kumo-custom-populations/skill-card.md | 2 +- skills/kumo-data-science/SKILL.md | 2 +- .../kumo-data-science/hosts/generic/runtime.md | 16 +++++++++++----- skills/kumo-data-science/skill-card.md | 12 ++++++------ skills/kumo-model-explanations/skill-card.md | 2 +- skills/kumo-model-persistence/skill-card.md | 6 +++--- skills/kumo-model-tuning/skill-card.md | 2 +- skills/kumo-training-diagnostics/skill-card.md | 2 +- skills/kumoai-reference/skill-card.md | 12 ++++++------ 17 files changed, 61 insertions(+), 49 deletions(-) diff --git a/agent-skills/nvskills/kumo-custom-populations/skill-card.md b/agent-skills/nvskills/kumo-custom-populations/skill-card.md index 8a1a901ae..fd07f006c 100644 --- a/agent-skills/nvskills/kumo-custom-populations/skill-card.md +++ b/agent-skills/nvskills/kumo-custom-populations/skill-card.md @@ -25,7 +25,7 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/populations.md](references/populations.md)
+- [references/populations.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-custom-populations/references/populations.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/agent-skills/nvskills/kumo-data-science/skill-card.md b/agent-skills/nvskills/kumo-data-science/skill-card.md index 9d4179590..f5c67fa46 100644 --- a/agent-skills/nvskills/kumo-data-science/skill-card.md +++ b/agent-skills/nvskills/kumo-data-science/skill-card.md @@ -25,12 +25,12 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [accept/intake_contract.md](accept/intake_contract.md)
-- [accept/preflight.md](accept/preflight.md)
-- [accept/batch_inference.md](accept/batch_inference.md)
-- [validate/atomic_question_check.md](validate/atomic_question_check.md)
-- [hosts/generic/adapter.md](hosts/generic/adapter.md)
-- [hosts/generic/runtime.md](hosts/generic/runtime.md)
+- [accept/intake_contract.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/intake_contract.md)
+- [accept/preflight.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/preflight.md)
+- [accept/batch_inference.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/batch_inference.md)
+- [validate/atomic_question_check.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/validate/atomic_question_check.md)
+- [hosts/generic/adapter.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/hosts/generic/adapter.md)
+- [hosts/generic/runtime.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/hosts/generic/runtime.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/agent-skills/nvskills/kumo-model-explanations/skill-card.md b/agent-skills/nvskills/kumo-model-explanations/skill-card.md index e231160f8..afea242ff 100644 --- a/agent-skills/nvskills/kumo-model-explanations/skill-card.md +++ b/agent-skills/nvskills/kumo-model-explanations/skill-card.md @@ -25,7 +25,7 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/artifacts.md](references/artifacts.md)
+- [references/artifacts.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-explanations/references/artifacts.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/agent-skills/nvskills/kumo-model-persistence/skill-card.md b/agent-skills/nvskills/kumo-model-persistence/skill-card.md index 1b02d2b88..30909eb4d 100644 --- a/agent-skills/nvskills/kumo-model-persistence/skill-card.md +++ b/agent-skills/nvskills/kumo-model-persistence/skill-card.md @@ -25,9 +25,9 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/reuse.md](references/reuse.md)
-- [references/retention.md](references/retention.md)
-- [scripts/retain_artifacts.py](scripts/retain_artifacts.py)
+- [references/reuse.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/references/reuse.md)
+- [references/retention.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/references/retention.md)
+- [scripts/retain_artifacts.py](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/scripts/retain_artifacts.py)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/agent-skills/nvskills/kumo-model-tuning/skill-card.md b/agent-skills/nvskills/kumo-model-tuning/skill-card.md index 9f36b8483..e78088763 100644 --- a/agent-skills/nvskills/kumo-model-tuning/skill-card.md +++ b/agent-skills/nvskills/kumo-model-tuning/skill-card.md @@ -25,7 +25,7 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/plans.md](references/plans.md)
+- [references/plans.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-tuning/references/plans.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/agent-skills/nvskills/kumo-training-diagnostics/skill-card.md b/agent-skills/nvskills/kumo-training-diagnostics/skill-card.md index d5e29a9e3..381026a7b 100644 --- a/agent-skills/nvskills/kumo-training-diagnostics/skill-card.md +++ b/agent-skills/nvskills/kumo-training-diagnostics/skill-card.md @@ -25,7 +25,7 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/diagnostics.md](references/diagnostics.md)
+- [references/diagnostics.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-training-diagnostics/references/diagnostics.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/agent-skills/nvskills/kumoai-reference/skill-card.md b/agent-skills/nvskills/kumoai-reference/skill-card.md index 438ebb66b..fdc400f7e 100644 --- a/agent-skills/nvskills/kumoai-reference/skill-card.md +++ b/agent-skills/nvskills/kumoai-reference/skill-card.md @@ -25,12 +25,12 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/graph_construction.md](references/graph_construction.md)
-- [references/pql_writing.md](references/pql_writing.md)
-- [references/kumo_finetuning_sdk.md](references/kumo_finetuning_sdk.md)
-- [references/task_journeys.md](references/task_journeys.md)
-- [references/outputs.md](references/outputs.md)
-- [references/troubleshooting_signals.md](references/troubleshooting_signals.md)
+- [references/graph_construction.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/graph_construction.md)
+- [references/pql_writing.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/pql_writing.md)
+- [references/kumo_finetuning_sdk.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/kumo_finetuning_sdk.md)
+- [references/task_journeys.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/task_journeys.md)
+- [references/outputs.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/outputs.md)
+- [references/troubleshooting_signals.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/troubleshooting_signals.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/agent-skills/skills/hosts/generic/runtime.md b/agent-skills/skills/hosts/generic/runtime.md index 2db65cdaa..2079db2ef 100644 --- a/agent-skills/skills/hosts/generic/runtime.md +++ b/agent-skills/skills/hosts/generic/runtime.md @@ -320,11 +320,13 @@ system; the destination may be in another system (`adapter.md` §4) and plays no schema = pa.schema([(keep[f.name], anchor_type(f.type) if keep[f.name] == 'TIMESTAMP' else f.type) for f in scanner.projected_schema]) - def to_column(col, field): # text in a non-ISO format parses with the entity's timestamp_format - if (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT - and (pa.types.is_string(col.type) or pa.types.is_large_string(col.type))): + def to_column(col, field): # a cast reads ISO-8601 text; other text needs the entity's format + try: + return pc.cast(col, field.type) + except pa.ArrowInvalid: + if not (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT): + raise return pc.strptime(col, format=ENTITY_TIME_FORMAT, unit=field.type.unit) - return pc.cast(col, field.type) pop_fs, pop_dir = arrow_fs(POPULATION_PATH) if isinstance(pop_fs, pyarrow.fs.LocalFileSystem): @@ -351,7 +353,11 @@ system; the destination may be in another system (`adapter.md` §4) and plays no `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. `ENTITY_TIME_FORMAT` is the `timestamp_format` the entity `Table` declares on its time column (`kumoai-reference/references/graph_construction.md`, *A time column must be a real date*), else - `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The population is + `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The cast runs + first, so ISO text staged as before, fractional seconds included; only text it cannot read is + parsed with the format. That format is Kumo's (chrono) syntax, and Arrow's parser rejects + `%.f`, `%-m` and `%:z`: if the parse then fails, report the format and the failing value rather + than guessing another format. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed it, both in the config cell. Each run replaces its `population.parquet`, as Node 1d says. The diff --git a/skills/kumo-custom-populations/skill-card.md b/skills/kumo-custom-populations/skill-card.md index 8a1a901ae..fd07f006c 100644 --- a/skills/kumo-custom-populations/skill-card.md +++ b/skills/kumo-custom-populations/skill-card.md @@ -25,7 +25,7 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/populations.md](references/populations.md)
+- [references/populations.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-custom-populations/references/populations.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index d03765cca..fe1d939dd 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-17-1dc2be3f`** +**`SKILL-BUILD: 2026-11-17-9126db08-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/hosts/generic/runtime.md b/skills/kumo-data-science/hosts/generic/runtime.md index 2db65cdaa..2079db2ef 100644 --- a/skills/kumo-data-science/hosts/generic/runtime.md +++ b/skills/kumo-data-science/hosts/generic/runtime.md @@ -320,11 +320,13 @@ system; the destination may be in another system (`adapter.md` §4) and plays no schema = pa.schema([(keep[f.name], anchor_type(f.type) if keep[f.name] == 'TIMESTAMP' else f.type) for f in scanner.projected_schema]) - def to_column(col, field): # text in a non-ISO format parses with the entity's timestamp_format - if (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT - and (pa.types.is_string(col.type) or pa.types.is_large_string(col.type))): + def to_column(col, field): # a cast reads ISO-8601 text; other text needs the entity's format + try: + return pc.cast(col, field.type) + except pa.ArrowInvalid: + if not (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT): + raise return pc.strptime(col, format=ENTITY_TIME_FORMAT, unit=field.type.unit) - return pc.cast(col, field.type) pop_fs, pop_dir = arrow_fs(POPULATION_PATH) if isinstance(pop_fs, pyarrow.fs.LocalFileSystem): @@ -351,7 +353,11 @@ system; the destination may be in another system (`adapter.md` §4) and plays no `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. `ENTITY_TIME_FORMAT` is the `timestamp_format` the entity `Table` declares on its time column (`kumoai-reference/references/graph_construction.md`, *A time column must be a real date*), else - `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The population is + `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The cast runs + first, so ISO text staged as before, fractional seconds included; only text it cannot read is + parsed with the format. That format is Kumo's (chrono) syntax, and Arrow's parser rejects + `%.f`, `%-m` and `%:z`: if the parse then fails, report the format and the failing value rather + than guessing another format. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed it, both in the config cell. Each run replaces its `population.parquet`, as Node 1d says. The diff --git a/skills/kumo-data-science/skill-card.md b/skills/kumo-data-science/skill-card.md index 9d4179590..f5c67fa46 100644 --- a/skills/kumo-data-science/skill-card.md +++ b/skills/kumo-data-science/skill-card.md @@ -25,12 +25,12 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [accept/intake_contract.md](accept/intake_contract.md)
-- [accept/preflight.md](accept/preflight.md)
-- [accept/batch_inference.md](accept/batch_inference.md)
-- [validate/atomic_question_check.md](validate/atomic_question_check.md)
-- [hosts/generic/adapter.md](hosts/generic/adapter.md)
-- [hosts/generic/runtime.md](hosts/generic/runtime.md)
+- [accept/intake_contract.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/intake_contract.md)
+- [accept/preflight.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/preflight.md)
+- [accept/batch_inference.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/batch_inference.md)
+- [validate/atomic_question_check.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/validate/atomic_question_check.md)
+- [hosts/generic/adapter.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/hosts/generic/adapter.md)
+- [hosts/generic/runtime.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/hosts/generic/runtime.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/skills/kumo-model-explanations/skill-card.md b/skills/kumo-model-explanations/skill-card.md index e231160f8..afea242ff 100644 --- a/skills/kumo-model-explanations/skill-card.md +++ b/skills/kumo-model-explanations/skill-card.md @@ -25,7 +25,7 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/artifacts.md](references/artifacts.md)
+- [references/artifacts.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-explanations/references/artifacts.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/skills/kumo-model-persistence/skill-card.md b/skills/kumo-model-persistence/skill-card.md index 1b02d2b88..30909eb4d 100644 --- a/skills/kumo-model-persistence/skill-card.md +++ b/skills/kumo-model-persistence/skill-card.md @@ -25,9 +25,9 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/reuse.md](references/reuse.md)
-- [references/retention.md](references/retention.md)
-- [scripts/retain_artifacts.py](scripts/retain_artifacts.py)
+- [references/reuse.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/references/reuse.md)
+- [references/retention.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/references/retention.md)
+- [scripts/retain_artifacts.py](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/scripts/retain_artifacts.py)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/skills/kumo-model-tuning/skill-card.md b/skills/kumo-model-tuning/skill-card.md index 9f36b8483..e78088763 100644 --- a/skills/kumo-model-tuning/skill-card.md +++ b/skills/kumo-model-tuning/skill-card.md @@ -25,7 +25,7 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/plans.md](references/plans.md)
+- [references/plans.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-tuning/references/plans.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/skills/kumo-training-diagnostics/skill-card.md b/skills/kumo-training-diagnostics/skill-card.md index d5e29a9e3..381026a7b 100644 --- a/skills/kumo-training-diagnostics/skill-card.md +++ b/skills/kumo-training-diagnostics/skill-card.md @@ -25,7 +25,7 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/diagnostics.md](references/diagnostics.md)
+- [references/diagnostics.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-training-diagnostics/references/diagnostics.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
diff --git a/skills/kumoai-reference/skill-card.md b/skills/kumoai-reference/skill-card.md index 438ebb66b..fdc400f7e 100644 --- a/skills/kumoai-reference/skill-card.md +++ b/skills/kumoai-reference/skill-card.md @@ -25,12 +25,12 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/graph_construction.md](references/graph_construction.md)
-- [references/pql_writing.md](references/pql_writing.md)
-- [references/kumo_finetuning_sdk.md](references/kumo_finetuning_sdk.md)
-- [references/task_journeys.md](references/task_journeys.md)
-- [references/outputs.md](references/outputs.md)
-- [references/troubleshooting_signals.md](references/troubleshooting_signals.md)
+- [references/graph_construction.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/graph_construction.md)
+- [references/pql_writing.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/pql_writing.md)
+- [references/kumo_finetuning_sdk.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/kumo_finetuning_sdk.md)
+- [references/task_journeys.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/task_journeys.md)
+- [references/outputs.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/outputs.md)
+- [references/troubleshooting_signals.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/troubleshooting_signals.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
From c7554e32540fe5a12cb3c77b6c575414d156bbcb Mon Sep 17 00:00:00 2001 From: igutierrez Date: Tue, 17 Nov 2026 19:53:01 -0600 Subject: [PATCH 23/35] agent-skills: quote BigQuery and Snowflake names in generated SQL NVSkills CI's Tier 1 security scan blocked the generic runtime: its read-back counts and the BigQuery population query pasted table and column names straight into SQL text. BigQuery names now go through `bq_name`, which backtick-quotes each part and refuses a part that could end the quoting. The Snowflake read-back goes through `sf_name`, which keeps database and schema unquoted as the connector does, but refuses anything but a plain identifier, and quotes the table. Co-Authored-By: Claude Opus 5.5 --- agent-skills/skills/hosts/generic/runtime.md | 41 ++++++++++++++----- skills/kumo-data-science/SKILL.md | 2 +- .../hosts/generic/runtime.md | 41 ++++++++++++++----- 3 files changed, 63 insertions(+), 21 deletions(-) diff --git a/agent-skills/skills/hosts/generic/runtime.md b/agent-skills/skills/hosts/generic/runtime.md index 2079db2ef..713485adf 100644 --- a/agent-skills/skills/hosts/generic/runtime.md +++ b/agent-skills/skills/hosts/generic/runtime.md @@ -102,8 +102,19 @@ def bq_location(project, dataset, credentials=None): """The location a dataset lives in; the dataset must already exist.""" return bigquery.Client(project=project, credentials=credentials).get_dataset( f'{project}.{dataset}').location + +def bq_name(*parts): + """`project`.`dataset`.`table`, or one `column`, for BigQuery SQL. A backtick, backslash or + line break is in no BigQuery name and would end the quoting, so such a part is refused.""" + for part in parts: + if not part or any(c in part for c in '`\\\n\r'): + raise ValueError(f'not a BigQuery name: {part!r}') + return '.'.join(f'`{part}`' for part in parts) ``` +Build every BigQuery table or column name in SQL with `bq_name`, never by pasting a name into the +query text. + Each BigQuery identity comes with the Google credentials its direct clients use: the location lookups, the population query (§3.1) and the read-back (§3). With an explicit `BigQueryIdentity` (*Credentials* below), build those credentials from the inputs the identity was given, as the SDK @@ -226,8 +237,8 @@ written = ds.dataset(OUTPUT_LOCATION, format=OUTPUT_FORMAT).count_rows() # full # BigQuery — as the destination's identity (§2) from google.cloud import bigquery bq = bigquery.Client(project=OUTPUT_PROJECT, credentials=OUTPUT_BQ_CREDENTIALS) -written = next(iter(bq.query( - f'SELECT COUNT(*) FROM `{OUTPUT_PROJECT}.{OUTPUT_DATASET}.{OUTPUT_TABLE}`').result()))[0] +count_sql = f'SELECT COUNT(*) FROM {bq_name(OUTPUT_PROJECT, OUTPUT_DATASET, OUTPUT_TABLE)}' +written = next(iter(bq.query(count_sql).result()))[0] # Databricks — with the confirmed SQL warehouse; wait_timeout is at most '50s', so poll import time @@ -250,12 +261,21 @@ written = int(r.result.data_array[0][0]) # Snowflake — through the same connections.toml entry and warehouse. Kumo keeps the table # name's case, so quote it (kumo_finetuning_sdk.md §9.1); database and schema as in §2. +import re import snowflake.connector + +def sf_name(database, schema, table): + """DATABASE.SCHEMA."table": database and schema unquoted, so they must be plain identifiers; + the table quoted, with any double quote in it doubled.""" + for part in (database, schema): + if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_$]*', part or ''): + raise ValueError(f'not an unquoted Snowflake name: {part!r}; confirm it with the user') + return f'{database}.{schema}."' + table.replace('"', '""') + '"' + with snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, warehouse=SNOWFLAKE_WAREHOUSE) as conn: - table = '"' + OUTPUT_TABLE.replace('"', '""') + '"' - written = conn.cursor().execute( - f'SELECT COUNT(*) FROM {OUTPUT_DATABASE}.{OUTPUT_SCHEMA}.{table}').fetchone()[0] + count_sql = f'SELECT COUNT(*) FROM {sf_name(OUTPUT_DATABASE, OUTPUT_SCHEMA, OUTPUT_TABLE)}' + written = conn.cursor().execute(count_sql).fetchone()[0] assert written - before == prediction_result.num_rows, (written, before, prediction_result.num_rows) ``` @@ -412,11 +432,12 @@ system; the destination may be in another system (`adapter.md` §4) and plays no if f'{POP_PROJECT}.{POP_DATASET}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') - time_col = f', CAST({ENTITY_TIME} AS TIMESTAMP) AS `TIMESTAMP`' if ENTITY_TIME else '' - bigquery.Client(project=POP_PROJECT, credentials=BQ_CREDENTIALS).query( # §2 - f'CREATE OR REPLACE TABLE `{POP_PROJECT}.{POP_DATASET}.{POP_TABLE}` AS ' - f'SELECT {ENTITY_KEY} AS ENTITY1{time_col} ' - f'FROM `{PROJECT_ID}.{DATASET_ID}.{ENTITY_TABLE}`').result() # waits; raises on failure + time_col = f', CAST({bq_name(ENTITY_TIME)} AS TIMESTAMP) AS `TIMESTAMP`' if ENTITY_TIME else '' + create_sql = (f'CREATE OR REPLACE TABLE {bq_name(POP_PROJECT, POP_DATASET, POP_TABLE)} AS ' + f'SELECT {bq_name(ENTITY_KEY)} AS ENTITY1{time_col} ' + f'FROM {bq_name(PROJECT_ID, DATASET_ID, ENTITY_TABLE)}') # bq_name: §2 + job = bigquery.Client(project=POP_PROJECT, credentials=BQ_CREDENTIALS).query(create_sql) # §2 + job.result() # waits; raises on failure population_connector = kumo.BigQueryConnector( project_id=POP_PROJECT, dataset_id=POP_DATASET, identity=BQ_IDENTITY, # the source's, §2 diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index fe1d939dd..bebf711ae 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-17-9126db08-dirty`** +**`SKILL-BUILD: 2026-11-17-53c38923-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/hosts/generic/runtime.md b/skills/kumo-data-science/hosts/generic/runtime.md index 2079db2ef..713485adf 100644 --- a/skills/kumo-data-science/hosts/generic/runtime.md +++ b/skills/kumo-data-science/hosts/generic/runtime.md @@ -102,8 +102,19 @@ def bq_location(project, dataset, credentials=None): """The location a dataset lives in; the dataset must already exist.""" return bigquery.Client(project=project, credentials=credentials).get_dataset( f'{project}.{dataset}').location + +def bq_name(*parts): + """`project`.`dataset`.`table`, or one `column`, for BigQuery SQL. A backtick, backslash or + line break is in no BigQuery name and would end the quoting, so such a part is refused.""" + for part in parts: + if not part or any(c in part for c in '`\\\n\r'): + raise ValueError(f'not a BigQuery name: {part!r}') + return '.'.join(f'`{part}`' for part in parts) ``` +Build every BigQuery table or column name in SQL with `bq_name`, never by pasting a name into the +query text. + Each BigQuery identity comes with the Google credentials its direct clients use: the location lookups, the population query (§3.1) and the read-back (§3). With an explicit `BigQueryIdentity` (*Credentials* below), build those credentials from the inputs the identity was given, as the SDK @@ -226,8 +237,8 @@ written = ds.dataset(OUTPUT_LOCATION, format=OUTPUT_FORMAT).count_rows() # full # BigQuery — as the destination's identity (§2) from google.cloud import bigquery bq = bigquery.Client(project=OUTPUT_PROJECT, credentials=OUTPUT_BQ_CREDENTIALS) -written = next(iter(bq.query( - f'SELECT COUNT(*) FROM `{OUTPUT_PROJECT}.{OUTPUT_DATASET}.{OUTPUT_TABLE}`').result()))[0] +count_sql = f'SELECT COUNT(*) FROM {bq_name(OUTPUT_PROJECT, OUTPUT_DATASET, OUTPUT_TABLE)}' +written = next(iter(bq.query(count_sql).result()))[0] # Databricks — with the confirmed SQL warehouse; wait_timeout is at most '50s', so poll import time @@ -250,12 +261,21 @@ written = int(r.result.data_array[0][0]) # Snowflake — through the same connections.toml entry and warehouse. Kumo keeps the table # name's case, so quote it (kumo_finetuning_sdk.md §9.1); database and schema as in §2. +import re import snowflake.connector + +def sf_name(database, schema, table): + """DATABASE.SCHEMA."table": database and schema unquoted, so they must be plain identifiers; + the table quoted, with any double quote in it doubled.""" + for part in (database, schema): + if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_$]*', part or ''): + raise ValueError(f'not an unquoted Snowflake name: {part!r}; confirm it with the user') + return f'{database}.{schema}."' + table.replace('"', '""') + '"' + with snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, warehouse=SNOWFLAKE_WAREHOUSE) as conn: - table = '"' + OUTPUT_TABLE.replace('"', '""') + '"' - written = conn.cursor().execute( - f'SELECT COUNT(*) FROM {OUTPUT_DATABASE}.{OUTPUT_SCHEMA}.{table}').fetchone()[0] + count_sql = f'SELECT COUNT(*) FROM {sf_name(OUTPUT_DATABASE, OUTPUT_SCHEMA, OUTPUT_TABLE)}' + written = conn.cursor().execute(count_sql).fetchone()[0] assert written - before == prediction_result.num_rows, (written, before, prediction_result.num_rows) ``` @@ -412,11 +432,12 @@ system; the destination may be in another system (`adapter.md` §4) and plays no if f'{POP_PROJECT}.{POP_DATASET}.{POP_TABLE}' != POPULATION_CONFIRMED: # approved at Node 1d raise RuntimeError('The population table changed after Node 1d: confirm the new one first.') - time_col = f', CAST({ENTITY_TIME} AS TIMESTAMP) AS `TIMESTAMP`' if ENTITY_TIME else '' - bigquery.Client(project=POP_PROJECT, credentials=BQ_CREDENTIALS).query( # §2 - f'CREATE OR REPLACE TABLE `{POP_PROJECT}.{POP_DATASET}.{POP_TABLE}` AS ' - f'SELECT {ENTITY_KEY} AS ENTITY1{time_col} ' - f'FROM `{PROJECT_ID}.{DATASET_ID}.{ENTITY_TABLE}`').result() # waits; raises on failure + time_col = f', CAST({bq_name(ENTITY_TIME)} AS TIMESTAMP) AS `TIMESTAMP`' if ENTITY_TIME else '' + create_sql = (f'CREATE OR REPLACE TABLE {bq_name(POP_PROJECT, POP_DATASET, POP_TABLE)} AS ' + f'SELECT {bq_name(ENTITY_KEY)} AS ENTITY1{time_col} ' + f'FROM {bq_name(PROJECT_ID, DATASET_ID, ENTITY_TABLE)}') # bq_name: §2 + job = bigquery.Client(project=POP_PROJECT, credentials=BQ_CREDENTIALS).query(create_sql) # §2 + job.result() # waits; raises on failure population_connector = kumo.BigQueryConnector( project_id=POP_PROJECT, dataset_id=POP_DATASET, identity=BQ_IDENTITY, # the source's, §2 From 9680c1ac53f065111c8ca807923ab2c5f0948b20 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Wed, 18 Nov 2026 02:56:50 +0000 Subject: [PATCH 24/35] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/kumo-custom-populations/BENCHMARK.md | 155 ++++++++++++++++ skills/kumo-custom-populations/skill-card.md | 71 +++++++- skills/kumo-custom-populations/skill.oms.sig | 1 + skills/kumo-data-science/BENCHMARK.md | 171 ++++++++++++++++++ skills/kumo-data-science/skill-card.md | 84 ++++++++- skills/kumo-data-science/skill.oms.sig | 1 + skills/kumo-graph-visualization/BENCHMARK.md | 155 ++++++++++++++++ skills/kumo-graph-visualization/skill-card.md | 75 +++++++- skills/kumo-graph-visualization/skill.oms.sig | 1 + skills/kumo-model-explanations/BENCHMARK.md | 157 ++++++++++++++++ skills/kumo-model-explanations/skill-card.md | 73 +++++++- skills/kumo-model-explanations/skill.oms.sig | 1 + skills/kumo-model-persistence/BENCHMARK.md | 157 ++++++++++++++++ skills/kumo-model-persistence/skill-card.md | 78 +++++++- skills/kumo-model-persistence/skill.oms.sig | 1 + skills/kumo-model-tuning/BENCHMARK.md | 155 ++++++++++++++++ skills/kumo-model-tuning/skill-card.md | 79 +++++++- skills/kumo-model-tuning/skill.oms.sig | 1 + skills/kumo-training-diagnostics/BENCHMARK.md | 155 ++++++++++++++++ .../kumo-training-diagnostics/skill-card.md | 87 ++++++++- .../kumo-training-diagnostics/skill.oms.sig | 1 + skills/kumoai-reference/BENCHMARK.md | 162 +++++++++++++++++ skills/kumoai-reference/skill-card.md | 83 ++++++++- skills/kumoai-reference/skill.oms.sig | 1 + 24 files changed, 1871 insertions(+), 34 deletions(-) create mode 100644 skills/kumo-custom-populations/BENCHMARK.md create mode 100644 skills/kumo-custom-populations/skill.oms.sig create mode 100644 skills/kumo-data-science/BENCHMARK.md create mode 100644 skills/kumo-data-science/skill.oms.sig create mode 100644 skills/kumo-graph-visualization/BENCHMARK.md create mode 100644 skills/kumo-graph-visualization/skill.oms.sig create mode 100644 skills/kumo-model-explanations/BENCHMARK.md create mode 100644 skills/kumo-model-explanations/skill.oms.sig create mode 100644 skills/kumo-model-persistence/BENCHMARK.md create mode 100644 skills/kumo-model-persistence/skill.oms.sig create mode 100644 skills/kumo-model-tuning/BENCHMARK.md create mode 100644 skills/kumo-model-tuning/skill.oms.sig create mode 100644 skills/kumo-training-diagnostics/BENCHMARK.md create mode 100644 skills/kumo-training-diagnostics/skill.oms.sig create mode 100644 skills/kumoai-reference/BENCHMARK.md create mode 100644 skills/kumoai-reference/skill.oms.sig diff --git a/skills/kumo-custom-populations/BENCHMARK.md b/skills/kumo-custom-populations/BENCHMARK.md new file mode 100644 index 000000000..387d62dd4 --- /dev/null +++ b/skills/kumo-custom-populations/BENCHMARK.md @@ -0,0 +1,155 @@ +# Skill Benchmark: kumo-custom-populations + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `kumo-custom-populations` +- Validation status: `passed` +- Evaluation date: 2026-11-18 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 5 evaluation tasks (4 positive, 1 negative) +- Dataset digest: `sha256:ed3c0e36c21c14588ddb79e7ed42113b76bddfa7ad9f90f3e27392b0fffdf2f2` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| kumo-custom-populations | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-custom-populations | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-custom-populations | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-custom-populations | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-custom-populations | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `kumo-custom-populations`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 92.9% — baseline ran, but no comparable score was available; uplift unavailable | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 56.0% → 100.0% (+44.0 points) | 64.0% → 92.0% (+28.0 points) | +| Discoverability | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 46.0% → 70.0% (+24.0 points) | 42.0% → 84.0% (+42.0 points) | +| Efficiency | 98.2% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 92.6% → 96.3% (+3.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 88.2% → 92.2% (+4.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 610,996 | 614,031 | -3,035 | -0.49% | skill 5/5; base 5/5 | +| claude-code | kpop-01 | 154,881 | 145,240 | +9,641 | +6.64% | skill 1/1; base 1/1 | +| claude-code | kpop-02 | 153,305 | 143,539 | +9,766 | +6.80% | skill 1/1; base 1/1 | +| claude-code | kpop-03 | 112,504 | 106,841 | +5,663 | +5.30% | skill 1/1; base 1/1 | +| claude-code | kpop-04 | 154,625 | 182,680 | -28,055 | -15.36% | skill 1/1; base 1/1 | +| claude-code | kpop-05 | 35,681 | 35,731 | -50 | -0.14% | skill 1/1; base 1/1 | +| codex | All cases | 192,572 | 257,130 | -64,558 | -25.11% | skill 5/5; base 5/5 | +| codex | kpop-01 | 43,740 | 66,096 | -22,356 | -33.82% | skill 1/1; base 1/1 | +| codex | kpop-02 | 43,769 | 45,971 | -2,202 | -4.79% | skill 1/1; base 1/1 | +| codex | kpop-03 | 43,649 | 45,724 | -2,075 | -4.54% | skill 1/1; base 1/1 | +| codex | kpop-04 | 43,767 | 81,593 | -37,826 | -46.36% | skill 1/1; base 1/1 | +| codex | kpop-05 | 17,647 | 17,746 | -99 | -0.56% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 803,568 | 871,161 | -67,593 | -7.76% | skill 10/10; base 10/10 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 11 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-custom-populations/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/kumo-custom-populations/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/kumo-custom-populations/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/kumo-custom-populations/SKILL.md`) +- **MEDIUM** SCHEMA/author_missing: Author not specified in metadata (`skills/kumo-custom-populations/SKILL.md`) +- 6 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/kumo-custom-populations/skill-card.md b/skills/kumo-custom-populations/skill-card.md index fd07f006c..10614fc17 100644 --- a/skills/kumo-custom-populations/skill-card.md +++ b/skills/kumo-custom-populations/skill-card.md @@ -16,16 +16,16 @@ Global
## Requirements / Dependencies:
**Requires API Key or External Credential:** [Yes]
-**Credential Type(s):** [Other [Snowflake connection or Snowpark session of the user's notebook]]
+**Credential Type(s):** [Other [Snowflake connection or Snowpark session of the user's notebook]]
-Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
## Known Risks and Mitigations:
Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/populations.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-custom-populations/references/populations.md)
+- [references/populations.md](references/populations.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
@@ -35,8 +35,71 @@ Mitigation: Review and scan skill before deployment.
**Output Parameters:** [1D]
**Other Properties Related to Output:** [Reads source tables only through read-only queries; never writes back to them.]
+## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod (`k8s-sandbox`); each task was also run without the skill as a baseline. Evaluator version `1.6.9`, evaluated 2026-11-18.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 92.9% — baseline ran, but no comparable score was available; uplift unavailable | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 56.0% → 100.0% (+44.0 points) | 64.0% → 92.0% (+28.0 points) | +| Discoverability | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 46.0% → 70.0% (+24.0 points) | 42.0% → 84.0% (+42.0 points) | +| Efficiency | 98.2% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 92.6% → 96.3% (+3.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 88.2% → 92.2% (+4.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + ## Skill Version(s):
-Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+c7554e32 (source: git SHA, committed 2026-11-17)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-custom-populations/skill.oms.sig b/skills/kumo-custom-populations/skill.oms.sig new file mode 100644 index 000000000..cbfb38cd5 --- /dev/null +++ b/skills/kumo-custom-populations/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1jdXN0b20tcG9wdWxhdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZjRjNGQ4Yzk3ZjA5MjQ2YzAwNGE2NmRjYzhmNjQ2MGQ4MTJkM2NmMTNiZTUzNGQwYWI1NGFkNzM4YzRkMzg4ZSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMTg1OTJiMjZlYTFiMDM2MjZiMjNmYTc5ZTc1YWMzNGEzNDdlYzcxYzExODk1ZTA0MzQwMTc0NDEyNjQ5YTBjMiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2MjI4MmZkMzY0MmI0MzMzNjY1NjJiYTU5YTUzYmE1YjkzMDI1N2U3M2UzYWQ1OGI5NmQ2OGMyNzAwZjM5NzU4IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzA1ZjU2YzBkNzJmMzViZWE5ZDIxMzBlNTYyZTUwZjk5YTEzYjkxMTZkOWY5ZDRkM2Q3ZTVmOTdkYWM5ZDdkOCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcG9wdWxhdGlvbnMubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjdmMjJmYjQzN2UwY2YwMGQxYzBjZWVlNjJhMGJiN2VmMGFmMzU0OGQzNDIyOWVmZmQxYzA1NWNiOWUyYWQyN2EiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIwZGQ5YWVlZjVhYjc4NGRlZmI4NGNhOTMxMTZkMzE1ZjliMWFlYzQzZTNhZDRiODIwNjYyOTlhMjJjZjI1NTNkIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQC8CrAmiDzpN+Dx+u+hxUj9EkPOZu5zmP0KIJfsljavnOblD1q9LlgMRJhlbbGQENkCMAC6JIPreLCihw8WDQIemS4AS4RHjmBdgRnvmAxQIUgPMRcnz7Cpmn4i68XHD3VFbw==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-data-science/BENCHMARK.md b/skills/kumo-data-science/BENCHMARK.md new file mode 100644 index 000000000..230d43550 --- /dev/null +++ b/skills/kumo-data-science/BENCHMARK.md @@ -0,0 +1,171 @@ +# Skill Benchmark: kumo-data-science + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `kumo-data-science` +- Validation status: `passed` +- Evaluation date: 2026-11-18 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 8 evaluation tasks (7 positive, 1 negative) +- Dataset digest: `sha256:f72dc6983fe9c6ff4f46b5d40bebd4ceb50356c04e83ebc88dc2cd383486d627` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| kumo-data-science | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-data-science | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-data-science | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-data-science | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-data-science | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `kumo-data-science`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 99.1% — baseline ran, but no comparable score was available; uplift unavailable | 94.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 93.8% → 100.0% (+6.2 points) | +| Correctness | 60.0% → 100.0% (+40.0 points) | 47.5% → 100.0% (+52.5 points) | +| Discoverability | 99.3% — baseline ran, but no comparable score was available; uplift unavailable | 90.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 39.3% → 100.0% (+60.7 points) | 38.8% → 85.9% (+47.1 points) | +| Efficiency | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.2% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 97.1% → 92.5% (-4.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 83.3% → 90.4% (+7.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 1,561,766 | 1,074,705 | +487,061 | +45.32% | skill 8/8; base 8/8 | +| claude-code | kds-01 | 227,358 | 143,260 | +84,098 | +58.70% | skill 1/1; base 1/1 | +| claude-code | kds-02 | 184,081 | 192,796 | -8,715 | -4.52% | skill 1/1; base 1/1 | +| claude-code | kds-03 | 336,605 | 273,297 | +63,308 | +23.16% | skill 1/1; base 1/1 | +| claude-code | kds-04 | 219,379 | 108,298 | +111,081 | +102.57% | skill 1/1; base 1/1 | +| claude-code | kds-05 | 172,823 | 106,455 | +66,368 | +62.34% | skill 1/1; base 1/1 | +| claude-code | kds-06 | 216,411 | 107,743 | +108,668 | +100.86% | skill 1/1; base 1/1 | +| claude-code | kds-07 | 169,197 | 107,191 | +62,006 | +57.85% | skill 1/1; base 1/1 | +| claude-code | kds-08 | 35,912 | 35,665 | +247 | +0.69% | skill 1/1; base 1/1 | +| codex | All cases | 617,755 | 3,018,694 | -2,400,939 | -79.54% | skill 8/8; base 8/8 | +| codex | kds-01 | 71,880 | 44,978 | +26,902 | +59.81% | skill 1/1; base 1/1 | +| codex | kds-02 | 97,244 | 1,714,885 | -1,617,641 | -94.33% | skill 1/1; base 1/1 | +| codex | kds-03 | 96,671 | 1,061,264 | -964,593 | -90.89% | skill 1/1; base 1/1 | +| codex | kds-04 | 117,090 | 41,458 | +75,632 | +182.43% | skill 1/1; base 1/1 | +| codex | kds-05 | 67,909 | 43,899 | +24,010 | +54.69% | skill 1/1; base 1/1 | +| codex | kds-06 | 84,028 | 41,072 | +42,956 | +104.59% | skill 1/1; base 1/1 | +| codex | kds-07 | 69,274 | 57,676 | +11,598 | +20.11% | skill 1/1; base 1/1 | +| codex | kds-08 | 13,659 | 13,462 | +197 | +1.46% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,179,521 | 4,093,399 | -1,913,878 | -46.76% | skill 16/16; base 16/16 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 27 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 4 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 8 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **HIGH** DUPLICATE/duplicate: Duplicate content found across SKILL.md and accept/preflight.md: + "### Node 1c — Preflight & graph gate" in SKILL.md (lines 178-194) + vs "## 5. Column availability" in accept/preflight.md (lines 157-171) (`SKILL.md:178`) +- **HIGH** DUPLICATE/duplicate: Duplicate content found across validate/atomic_question_check.md and validate/examples/bad_request.md: + "### Fail 1 — Multiple targets bundled" in validate/atomic_question_check.md (lines 42-50) + vs "## 6. Bundled goals → REJECT" in validate/examples/bad_request.md (lines 64-68) (`validate/atomic_question_check.md:42`) +- **HIGH** DUPLICATE/duplicate: Duplicate content found across accept/intake_contract.md and validate/examples/good_request.md: + "## Canonical good example (`train`)" in accept/intake_contract.md (lines 137-148) + vs "## Input" in validate/examples/good_request.md (lines 3-12) + vs "## Node 1a — Required fields: PASS" in validate/examples/good_request.md (lines 13-19) (`accept/intake_contract.md:137`) +- **HIGH** DUPLICATE/duplicate: Duplicate content found within accept/batch_inference.md: + "## Node 1a — fields" in accept/batch_inference.md (lines 7-29) + vs "### `batch_inference` — required fields (ask if missing)" in accept/batch_inference.md (lines 30-42) + vs "### `batch_inference` — resolved, defaulted, or inferred (never ask)" in accept/batch_inference.md (lines 43-54) (`accept/batch_inference.md:7`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-data-science/SKILL.md`) +- 26 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/kumo-data-science/skill-card.md b/skills/kumo-data-science/skill-card.md index f5c67fa46..50954d964 100644 --- a/skills/kumo-data-science/skill-card.md +++ b/skills/kumo-data-science/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Turns a plain-language request for a KumoAI predictive model on relational data into a confirmed task and graph through a multi-turn intake, generates one notebook or script that trains and predicts with the local Kumo SDK, and routes inspection, persistence, population and tuning requests to their own Kumo skills.
+Turns a plain-language request for a KumoAI predictive model on relational data in a warehouse, an object store or local files into a confirmed task and graph through a multi-turn intake, generates one notebook or script that trains and predicts with the local Kumo SDK (running it only when asked), and routes graph inspection, explanation, diagnostics, persistence, custom-population and tuning requests to their own Kumo skills.
This skill is ready for commercial/non-commercial use.
@@ -9,16 +9,16 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill with coding agents to build Kumo models for churn, propensity, fraud, recommendation, forecasting and link prediction on data in Snowflake, BigQuery, Databricks, S3, GCS or local files: the agent checks the request, proposes the graph for confirmation and writes the training-and-prediction notebook.
+Developers and data scientists use this skill with coding agents to build Kumo models for churn, propensity, fraud, recommendation, forecasting and link prediction on relational data in Snowflake, BigQuery, Databricks, S3, GCS or local files: the agent checks the request, proposes the graph and task for confirmation and writes the training-and-prediction notebook, or helps create and debug the jobs that run it.
### Deployment Geography for Use:
Global
## Requirements / Dependencies:
**Requires API Key or External Credential:** [Optional]
-**Credential Type(s):** [Cloud Credentials, OAuth Token, Other [Snowflake connections.toml entry]]
+**Credential Type(s):** [Cloud Credentials, Service Account, Other [Snowflake connections.toml entry]]
-Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
## Known Risks and Mitigations:
Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
@@ -28,9 +28,12 @@ Mitigation: Review and scan skill before deployment.
- [accept/intake_contract.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/intake_contract.md)
- [accept/preflight.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/preflight.md)
- [accept/batch_inference.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/batch_inference.md)
+- [accept/semantic_view.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/semantic_view.md)
- [validate/atomic_question_check.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/validate/atomic_question_check.md)
- [hosts/generic/adapter.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/hosts/generic/adapter.md)
- [hosts/generic/runtime.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/hosts/generic/runtime.md)
+- [NVIDIA/kumo-engine#134 (graph visualization)](https://github.com/NVIDIA/kumo-engine/pull/134)
+- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
@@ -40,8 +43,79 @@ Mitigation: Review and scan skill before deployment.
**Output Parameters:** [1D]
**Other Properties Related to Output:** [Generates code only; runs it only when the user asks. Writes predictions only to a destination the user confirmed.]
+## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+8 evaluation tasks (7 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.9`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 99.1% — baseline ran, but no comparable score was available; uplift unavailable | 94.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 93.8% → 100.0% (+6.2 points) | +| Correctness | 60.0% → 100.0% (+40.0 points) | 47.5% → 100.0% (+52.5 points) | +| Discoverability | 99.3% — baseline ran, but no comparable score was available; uplift unavailable | 90.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 39.3% → 100.0% (+60.7 points) | 38.8% → 85.9% (+47.1 points) | +| Efficiency | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.2% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 97.1% → 92.5% (-4.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 83.3% → 90.4% (+7.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 27 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 4 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 8 task(s) | + ## Skill Version(s):
-Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+c7554e32 (source: git SHA, committed 2026-11-17); repository pyproject.toml version 0.1.0; SKILL.md build marker SKILL-BUILD: 2026-11-17-53c38923-dirty
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-data-science/skill.oms.sig b/skills/kumo-data-science/skill.oms.sig new file mode 100644 index 000000000..8e25c5ce5 --- /dev/null +++ b/skills/kumo-data-science/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1kYXRhLXNjaWVuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYzQ5NmE3MGI1Zjk1NTE4ZjFhMTI0MTI0ODA0NmMxMmM4MDkxNjVlYWY2ZjcyNzQ2Yzk1NTkwMWY5N2NkOTkzOSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZjQzN2VjOWFhZjJiMDIzZGRhYWU0MzM0M2MzNmQzNDI3NWE0ZDNiZDgxNmIwNjUwYWZjOTZjZDI2ZjcxZjE3OSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMjM0YjAxOTU3MGIyMGJmNGJkZmQ3ZGU2YjNlYzU4ODJiYmUwMTg2Y2FiYjVmZDY4NDEzOTRmOTg5MzRlZDZjMCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIyZDc1YWJhMGZhNGZlMTliODg1NjUzMjgwZTFhNTJkMTNiMDdhMGUxMzNkYzQzNGQxYjJiZDdhYjBmOWY2YmI4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L2JhdGNoX2luZmVyZW5jZS5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZmVhN2JiZDg2NWI4MzY2NDExMDdmNzQwMzQ4MjZiMWY1NGExNGQxNGYzOGIxN2VhYzBmZGRhODA4OTQxMjJkMSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFjY2VwdC9pbnRha2VfY29udHJhY3QubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImQzYjRjOTA3NDE1ZWE0MzYwZTA5YTRhYTIyZTljYThlNjhiZDIzZjhiNGM0N2NjZjlkMWI5NWQ4YWE2MWJlOWYiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJhY2NlcHQvcHJlZmxpZ2h0Lm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJmOWYyZGY5Y2FhNGQ1NjllNDNlNGQzNGY3MDk5YjIxNTExZmFjMWFiMzllZDYyMjczMjQ4ZmI2MTFiOWNiZDNlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L3NlbWFudGljX3ZpZXcubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjY2NDA0ZDljYjEzZDAwOTNiNjcwMWRmZDRiMTkzMzcxMDNkOGQzYjg2MDc1ZTExMTM5MzZkM2I0NGU5OWNjYTkiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJjNmEzNjg0NTRmYTlhZGVlYmEwYmRkYTk4YmQzZDIxNGYyYjkyYTVmMTI4MzUyNzQ1YzNkMjJlYjkwN2I1YjZiIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiaG9zdHMvZ2VuZXJpYy9hZGFwdGVyLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIxM2I3Y2UwMmU3MjU1YzY3ODNiODVlYWJhMmVjYThiMWY5ODU3YWUxM2FiNDQ3ZWI1NDdmZjdhMWExM2YyMjNjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiaG9zdHMvZ2VuZXJpYy9ydW50aW1lLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJiM2Q0YmUwYjNiNTQzN2E0ZTc4MTg1NTNkNmJkNzhlOGExYTBlODgyNDc5NTYxYmUzYWMxMGIwODZjODc2M2Y3IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNjUwNmU2NjFiYzExYzI1ZWI2NmE3YjdjMzVhMzQwMzEzOTU3ZDMyNGJhMzg2N2U5NTA4NmIxMTRlOTJkMjA0YSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2F0b21pY19xdWVzdGlvbl9jaGVjay5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiN2JkMzkzNGUxYjljMjJhNGMzOGIzN2JjYjhjNTk3MjkyZGRlZTgyMjA2MWQ5ZmVlZWU2NTdiZDkwNTAzNWNkNCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2V4YW1wbGVzL2JhZF9yZXF1ZXN0Lm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI4OTJkMGZiZWI0MmE4MDJkMzM2ZmZiMjMwOWQ1ZTk5OWIzOGNmYjQ4N2QzNjljMzg5Mjk0OWYzY2MxYmVkOGU2IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAidmFsaWRhdGUvZXhhbXBsZXMvZ29vZF9yZXF1ZXN0Lm1kIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQC0C2zqrFD3/TElDbjwOz15rlNcHqRk/J8ka9DCxR/Jg0UhKcOvpTKUNNMpqO0F+FMCMGQmSWA3F7fo3UcG3u1osYLviYg2WoHqos9oIjjx2QrYFRqJEygorvRo6TwZwiIcbw==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-graph-visualization/BENCHMARK.md b/skills/kumo-graph-visualization/BENCHMARK.md new file mode 100644 index 000000000..a5513ca7a --- /dev/null +++ b/skills/kumo-graph-visualization/BENCHMARK.md @@ -0,0 +1,155 @@ +# Skill Benchmark: kumo-graph-visualization + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `kumo-graph-visualization` +- Validation status: `passed` +- Evaluation date: 2026-11-18 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 5 evaluation tasks (4 positive, 1 negative) +- Dataset digest: `sha256:5ce8bff0dfb494254338b949f512aa368984b20ef512d553ecdf7175e11ea4c8` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| kumo-graph-visualization | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-graph-visualization | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-graph-visualization | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-graph-visualization | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-graph-visualization | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `kumo-graph-visualization`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 88.1% — baseline ran, but no comparable score was available; uplift unavailable | 79.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 68.0% (+40.0 points) | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 30.8% → 50.0% (+19.2 points) | 30.0% → 37.5% (+7.5 points) | +| Efficiency | 91.5% — baseline ran, but no comparable score was available; uplift unavailable | 96.5% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 89.9% → 97.7% (+7.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 91.3% → 93.0% (+1.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 804,052 | 715,405 | +88,647 | +12.39% | skill 5/5; base 5/5 | +| claude-code | kviz-01 | 145,945 | 143,072 | +2,873 | +2.01% | skill 1/1; base 1/1 | +| claude-code | kviz-02 | 147,079 | 107,442 | +39,637 | +36.89% | skill 1/1; base 1/1 | +| claude-code | kviz-03 | 185,748 | 179,161 | +6,587 | +3.68% | skill 1/1; base 1/1 | +| claude-code | kviz-04 | 110,109 | 107,697 | +2,412 | +2.24% | skill 1/1; base 1/1 | +| claude-code | kviz-05 | 215,171 | 178,033 | +37,138 | +20.86% | skill 1/1; base 1/1 | +| codex | All cases | 241,390 | 257,645 | -16,255 | -6.31% | skill 5/5; base 5/5 | +| codex | kviz-01 | 57,543 | 68,459 | -10,916 | -15.95% | skill 1/1; base 1/1 | +| codex | kviz-02 | 27,456 | 46,484 | -19,028 | -40.93% | skill 1/1; base 1/1 | +| codex | kviz-03 | 57,025 | 40,586 | +16,439 | +40.50% | skill 1/1; base 1/1 | +| codex | kviz-04 | 42,204 | 61,244 | -19,040 | -31.09% | skill 1/1; base 1/1 | +| codex | kviz-05 | 57,162 | 40,872 | +16,290 | +39.86% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,045,442 | 973,050 | +72,392 | +7.44% | skill 10/10; base 10/10 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 10 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-graph-visualization/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/kumo-graph-visualization/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/kumo-graph-visualization/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/kumo-graph-visualization/SKILL.md`) +- **MEDIUM** SCHEMA/author_missing: Author not specified in metadata (`skills/kumo-graph-visualization/SKILL.md`) +- 5 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/kumo-graph-visualization/skill-card.md b/skills/kumo-graph-visualization/skill-card.md index 9eed168ad..f358e0ebc 100644 --- a/skills/kumo-graph-visualization/skill-card.md +++ b/skills/kumo-graph-visualization/skill-card.md @@ -16,9 +16,9 @@ Global
## Requirements / Dependencies:
**Requires API Key or External Credential:** [Not Specified]
-**Credential Type(s):** [None identified]
+**Credential Type(s):** [None identified]
-Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
## Known Risks and Mitigations:
Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
@@ -26,6 +26,7 @@ Mitigation: Review and scan skill before deployment.
## Reference(s):
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
## Skill Output:
@@ -34,8 +35,76 @@ Mitigation: Review and scan skill before deployment.
**Output Parameters:** [1D]
**Other Properties Related to Output:** [Source sample values are off unless the user asks for them.]
+## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+5 evaluation tasks (4 positive, 1 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`), each compared against a no-skill baseline. Evaluator version `1.6.9`, evaluation date 2026-11-18.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 88.1% — baseline ran, but no comparable score was available; uplift unavailable | 79.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 68.0% (+40.0 points) | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 30.8% → 50.0% (+19.2 points) | 30.0% → 37.5% (+7.5 points) | +| Efficiency | 91.5% — baseline ran, but no comparable score was available; uplift unavailable | 96.5% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 89.9% → 97.7% (+7.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 91.3% → 93.0% (+1.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 10 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | + +**Overall verdict:** PASS — Recommended for publication
+**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report.
+ ## Skill Version(s):
-Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+0.1.0 (source: pyproject.toml; git SHA c7554e32, committed 2026-11-17)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-graph-visualization/skill.oms.sig b/skills/kumo-graph-visualization/skill.oms.sig new file mode 100644 index 000000000..83bbbc5b4 --- /dev/null +++ b/skills/kumo-graph-visualization/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1ncmFwaC12aXN1YWxpemF0aW9uIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImJlZmMwZTkyOTE5MDAwM2U1YWRmZWU4ODUyYzMzY2Q3ZmJmOTQ3NzMyOWVmNDE1YWMxMTA4ZWQ3OTliMzRlMDUiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0KICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMDg2NjhmNTg3NjdlY2Q5Njk2NDc5ODg4YTZkMTAyYzA1OWY1NzY4ZjhjOGNiZGE4ZGFlNDI5YzY3YzMwOWJhYSIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOTU2M2MyZDVkNGQzMmJhYjAyOGE0Zjk2YmExNWU0NDVkN2FlYzIyZjEzZjRkYmY0NDc2NWVjZjUzYjBjOGZmNyIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3NzRhMjc4YTY2OTBmZTBiZjcxMDAxZDc2MDk1ZDM2NmQxMGE4MDNjOGE2Njg2NWRkZjM0YTMyZTc1ZTZiOTliIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNGEzMThkYTdmZmMxOTI3MTgzOTQ4NmJkZDE1NDNkMjhhY2NmMmM2MGEyZGU0NDVmMTdkNGQzYWU0YTJlMGVmOSIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMEX2cvnKYaf3YEDmv4qUDYx2WGb13J1tEJ+zljdQP9azsUzwvPKTmxIBmJpqXD1ecAIwUqZeNXxq8M1N360XeASg5Xdmq6LoM3W3LH1X/oEoBFgF76HoFD07ZGVSrDVR2e42","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-explanations/BENCHMARK.md b/skills/kumo-model-explanations/BENCHMARK.md new file mode 100644 index 000000000..ac49beb9d --- /dev/null +++ b/skills/kumo-model-explanations/BENCHMARK.md @@ -0,0 +1,157 @@ +# Skill Benchmark: kumo-model-explanations + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `kumo-model-explanations` +- Validation status: `passed` +- Evaluation date: 2026-11-18 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 6 evaluation tasks (5 positive, 1 negative) +- Dataset digest: `sha256:e389a4317a270919c8e56c8d8f40c51460e1ecc5bef453075887cea72bad3559` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| kumo-model-explanations | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-explanations | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-explanations | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-explanations | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-explanations | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `kumo-model-explanations`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 91.7% — baseline ran, but no comparable score was available; uplift unavailable | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 46.7% → 100.0% (+53.3 points) | 63.3% → 93.3% (+30.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 29.2% → 60.4% (+31.2 points) | 41.7% → 57.0% (+15.3 points) | +| Efficiency | 97.9% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 95.8% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 88.9% → 92.0% (+3.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 927,473 | 787,744 | +139,729 | +17.74% | skill 6/6; base 6/6 | +| claude-code | kexpl-01 | 158,137 | 142,432 | +15,705 | +11.03% | skill 1/1; base 1/1 | +| claude-code | kexpl-02 | 159,124 | 108,697 | +50,427 | +46.39% | skill 1/1; base 1/1 | +| claude-code | kexpl-03 | 115,976 | 143,259 | -27,283 | -19.04% | skill 1/1; base 1/1 | +| claude-code | kexpl-04 | 159,980 | 107,413 | +52,567 | +48.94% | skill 1/1; base 1/1 | +| claude-code | kexpl-05 | 226,109 | 178,383 | +47,726 | +26.75% | skill 1/1; base 1/1 | +| claude-code | kexpl-06 | 108,147 | 107,560 | +587 | +0.55% | skill 1/1; base 1/1 | +| codex | All cases | 410,225 | 297,492 | +112,733 | +37.89% | skill 6/6; base 6/6 | +| codex | kexpl-01 | 81,280 | 43,931 | +37,349 | +85.02% | skill 1/1; base 1/1 | +| codex | kexpl-02 | 83,129 | 53,527 | +29,602 | +55.30% | skill 1/1; base 1/1 | +| codex | kexpl-03 | 80,951 | 60,563 | +20,388 | +33.66% | skill 1/1; base 1/1 | +| codex | kexpl-04 | 81,288 | 57,670 | +23,618 | +40.95% | skill 1/1; base 1/1 | +| codex | kexpl-05 | 42,650 | 41,123 | +1,527 | +3.71% | skill 1/1; base 1/1 | +| codex | kexpl-06 | 40,927 | 40,678 | +249 | +0.61% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,337,698 | 1,085,236 | +252,462 | +23.26% | skill 12/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-model-explanations/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/kumo-model-explanations/SKILL.md`) +- **MEDIUM** QUALITY/quality_efficiency: Deeply nested references in artifacts.md (`skills/kumo-model-explanations/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/kumo-model-explanations/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/kumo-model-explanations/SKILL.md`) +- 9 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/kumo-model-explanations/skill-card.md b/skills/kumo-model-explanations/skill-card.md index afea242ff..f9bf69206 100644 --- a/skills/kumo-model-explanations/skill-card.md +++ b/skills/kumo-model-explanations/skill-card.md @@ -16,9 +16,9 @@ Global
## Requirements / Dependencies:
**Requires API Key or External Credential:** [Not Specified]
-**Credential Type(s):** [None identified]
+**Credential Type(s):** [None identified]
-Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
## Known Risks and Mitigations:
Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
@@ -35,8 +35,75 @@ Mitigation: Review and scan skill before deployment.
**Output Parameters:** [1D]
**Other Properties Related to Output:** [Inspection only; makes no causal claims and starts no fit.]
+## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.9`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 91.7% — baseline ran, but no comparable score was available; uplift unavailable | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 46.7% → 100.0% (+53.3 points) | 63.3% → 93.3% (+30.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 29.2% → 60.4% (+31.2 points) | 41.7% → 57.0% (+15.3 points) | +| Efficiency | 97.9% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 95.8% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 88.9% → 92.0% (+3.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + ## Skill Version(s):
-Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+c7554e32 (source: git SHA, committed 2026-11-17); repository pyproject.toml version 0.1.0
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-model-explanations/skill.oms.sig b/skills/kumo-model-explanations/skill.oms.sig new file mode 100644 index 000000000..e832b2c04 --- /dev/null +++ b/skills/kumo-model-explanations/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1leHBsYW5hdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYWQzOGI1NDBhMTU0MTA2MjZlY2NjN2NkNzFmMWI5YjM2NDZhZDFjZTQzNzFhYjA4YWYyZmVkZTY5MmQxYmNmYyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzYzRmZmQyNTJjYWY3Y2E1N2I3ZWQzMDQ2MjUzZTA3YTMzZGEwNDZmMjI1MmY1ZDYwZjlmMDIyOThiNTRhNTEyIiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJjYjc0YjdkZGNkZDNhZTA2YzUyYmM4NTJjZDljOTc5ZTk0YmE2NDBjNmVjZTgyMmY1ZjQ5MmJiZmY5N2RhYTBjIiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjk2NzRhNzFhMWExYmYwNmU1ZjNhMDdlNGNjZWRlNTdkMTMzNjYzOWY0NGMyMWYwMzQzYWQ3MmE3ZTA3MmRiMTciLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI1MGNiNjhmYWJkNzRjOGUxZTkyYzRjODMwZWU1MDllZGNiNjY1NGI3MGY5OGMyN2NlOGZlMzRjODk4ZWE3MTdhIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2FydGlmYWN0cy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjNlOTU4ZGJiNWVlOWQ1YjA1YjRjMTc2NDk0NjE3NWU3YmUxZjI3NTFhY2U3NDFiMmE2MDJlYzY3NTEwZjQyZjMiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMD2y1HVohgd0Hadvyl6V8B2Kmycfk+L30bDQr8g9KBCdWzvzYR+9cBfzU5BvwsGg4gIwb4vOX1RgNRbH/tV4PrMa2l33SChf4K47ASVVXTu9NrcQiWpxgMId+e5yyoJiA8oH","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-persistence/BENCHMARK.md b/skills/kumo-model-persistence/BENCHMARK.md new file mode 100644 index 000000000..8aac466ed --- /dev/null +++ b/skills/kumo-model-persistence/BENCHMARK.md @@ -0,0 +1,157 @@ +# Skill Benchmark: kumo-model-persistence + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `kumo-model-persistence` +- Validation status: `passed` +- Evaluation date: 2026-11-18 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 6 evaluation tasks (5 positive, 1 negative) +- Dataset digest: `sha256:096c403e279883d033c46cf48b03d51e3a6ade8a0c67149a36066124537ddb47` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| kumo-model-persistence | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-persistence | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-persistence | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-persistence | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-persistence | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `kumo-model-persistence`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 50.0% → 96.7% (+46.7 points) | 36.7% → 76.7% (+40.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 35.0% → 47.2% (+12.2 points) | 13.9% → 61.1% (+47.2 points) | +| Efficiency | 98.4% — baseline ran, but no comparable score was available; uplift unavailable | 91.3% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 97.9% → 96.8% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 91.4% → 92.3% (+0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 1,019,743 | 830,788 | +188,955 | +22.74% | skill 6/6; base 6/6 | +| claude-code | kpersist-01 | 189,044 | 144,899 | +44,145 | +30.47% | skill 1/1; base 1/1 | +| claude-code | kpersist-02 | 188,504 | 184,161 | +4,343 | +2.36% | skill 1/1; base 1/1 | +| claude-code | kpersist-03 | 194,119 | 143,156 | +50,963 | +35.60% | skill 1/1; base 1/1 | +| claude-code | kpersist-04 | 269,404 | 144,174 | +125,230 | +86.86% | skill 1/1; base 1/1 | +| claude-code | kpersist-05 | 71,208 | 106,978 | -35,770 | -33.44% | skill 1/1; base 1/1 | +| claude-code | kpersist-06 | 107,464 | 107,420 | +44 | +0.04% | skill 1/1; base 1/1 | +| codex | All cases | 445,785 | 303,864 | +141,921 | +46.71% | skill 6/6; base 6/6 | +| codex | kpersist-01 | 93,061 | 41,296 | +51,765 | +125.35% | skill 1/1; base 1/1 | +| codex | kpersist-02 | 42,478 | 41,293 | +1,185 | +2.87% | skill 1/1; base 1/1 | +| codex | kpersist-03 | 131,524 | 55,298 | +76,226 | +137.85% | skill 1/1; base 1/1 | +| codex | kpersist-04 | 95,555 | 64,736 | +30,819 | +47.61% | skill 1/1; base 1/1 | +| codex | kpersist-05 | 42,036 | 60,287 | -18,251 | -30.27% | skill 1/1; base 1/1 | +| codex | kpersist-06 | 41,131 | 40,954 | +177 | +0.43% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,465,528 | 1,134,652 | +330,876 | +29.16% | skill 12/12; base 12/12 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 22 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: No documented scripts in table format (`skills/kumo-model-persistence/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: Instructions don't mention 'run_script' (`skills/kumo-model-persistence/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-model-persistence/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/kumo-model-persistence/SKILL.md`) +- **MEDIUM** QUALITY/quality_efficiency: Deeply nested references in retention.md (`skills/kumo-model-persistence/SKILL.md`) +- 17 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/kumo-model-persistence/skill-card.md b/skills/kumo-model-persistence/skill-card.md index 30909eb4d..ee47bf529 100644 --- a/skills/kumo-model-persistence/skill-card.md +++ b/skills/kumo-model-persistence/skill-card.md @@ -16,9 +16,9 @@ Global
## Requirements / Dependencies:
**Requires API Key or External Credential:** [Yes]
-**Credential Type(s):** [Other [Snowflake connection or Snowpark session of the user's notebook]]
+**Credential Type(s):** [Other [Snowflake connection or Snowpark session of the user's notebook]]
-Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
## Known Risks and Mitigations:
Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
@@ -28,6 +28,7 @@ Mitigation: Review and scan skill before deployment.
- [references/reuse.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/references/reuse.md)
- [references/retention.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/references/retention.md)
- [scripts/retain_artifacts.py](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/scripts/retain_artifacts.py)
+- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
@@ -37,8 +38,79 @@ Mitigation: Review and scan skill before deployment.
**Output Parameters:** [1D]
**Other Properties Related to Output:** [Loading a model does not authorize training, replacing a registry version or overwriting a destination.]
+## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.9`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 50.0% → 96.7% (+46.7 points) | 36.7% → 76.7% (+40.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 35.0% → 47.2% (+12.2 points) | 13.9% → 61.1% (+47.2 points) | +| Efficiency | 98.4% — baseline ran, but no comparable score was available; uplift unavailable | 91.3% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 97.9% → 96.8% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 91.4% → 92.3% (+0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 22 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | + ## Skill Version(s):
-Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+c7554e32 (source: git SHA, committed 2026-11-17); repository pyproject.toml version 0.1.0
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-model-persistence/skill.oms.sig b/skills/kumo-model-persistence/skill.oms.sig new file mode 100644 index 000000000..34f014cac --- /dev/null +++ b/skills/kumo-model-persistence/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1wZXJzaXN0ZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIxNWZhOWE5Y2Q0ZDkxZDA4MDc4OTI0NGY3NjgxZTYxNzc0MzBjMjIxNGM2Yzc3ZTUyOGE1MzEwZmRjNGU4MzBhIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiMjY3ZmMzNDFhNjZmZmU3NjUxNmRmNDY4MDYwMzhkMmY1ZDY2YWIyMzJiZTU1ZDI0Nzc0ZGZlMWM4YTVhZTdlYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJiMWU4MWU2ODg2ZDNkYTczZmMwMzI2ZjhmMGQ1YTYzYTBjNzIzNjlkNmYyNWZkM2M0MjkyN2U5YWUyMDBlZTI4IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiMzY3ZTg3MWE3MmUzZTZjN2I1MWRhYjlhMzJiMzMwOTQyZDAwMjllZTY0NzNjMzM3NjcyMzI2NGQyMmYzY2E2YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3JldGVudGlvbi5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIyYjA1NGFmMzNmODE2MmM1MThkZWE5ZjdhZDg0NGMwZDQxNjA3YjAwM2Q1ODIxZDUwYTJiZjY4YmU2Y2FjZjMyIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcmV1c2UubWQiLAogICAgICAgICJkaWdlc3QiOiAiMmM1YTUwMzFjODUwMTlmMTIxNWU2Yzc0MGRjZWRlN2E2ZTkyMDg1MmIzZjU3MmYwNGY5NGRhMGVjY2M4YWU4MSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JldGFpbl9hcnRpZmFjdHMucHkiLAogICAgICAgICJkaWdlc3QiOiAiNGQzZDAzYjEwODlhMDI0MGQxZjkxM2MxZWQyZjhlNGFmOTg5ZWNiOGQzZTE5OWMzMmZkZGFlOTBmY2JhYWQzYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjg4OGExNDcwM2QyNTQ3OWJjZjYwYjhjMzI0ZWI0MTI3ZGIyYjgwMTIwM2JjMmNiYjhjMGYwYzhmZTBhZTRmNjgiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMGhSpnlbYzj/ttppt3sbMtkii3c2wBbfu8/Z0P1RGuf5UefvzGs4e2sFpJttI8QTFwIwe8FOIzE0F9lUuIJS90dU79X/otq5xuHnj7f+bEgT/pIzKv2X8k6bK0WqI2LqN9ds","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-tuning/BENCHMARK.md b/skills/kumo-model-tuning/BENCHMARK.md new file mode 100644 index 000000000..57f3c6fc7 --- /dev/null +++ b/skills/kumo-model-tuning/BENCHMARK.md @@ -0,0 +1,155 @@ +# Skill Benchmark: kumo-model-tuning + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `kumo-model-tuning` +- Validation status: `passed` +- Evaluation date: 2026-11-18 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 5 evaluation tasks (4 positive, 1 negative) +- Dataset digest: `sha256:7fd78018aceb270831a95ad2a267d9e84f1e0175652850059d7cda7224f540ae` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| kumo-model-tuning | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-tuning | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-tuning | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-tuning | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-tuning | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `kumo-model-tuning`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 96.7% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 60.0% → 96.0% (+36.0 points) | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 47.5% → 90.0% (+42.5 points) | 51.7% → 80.8% (+29.1 points) | +| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 97.3% (-0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 91.6% → 92.3% (+0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 808,588 | 831,826 | -23,238 | -2.79% | skill 5/5; base 5/5 | +| claude-code | ktune-01 | 109,085 | 71,246 | +37,839 | +53.11% | skill 1/1; base 1/1 | +| claude-code | ktune-02 | 147,053 | 142,382 | +4,671 | +3.28% | skill 1/1; base 1/1 | +| claude-code | ktune-03 | 147,632 | 178,628 | -30,996 | -17.35% | skill 1/1; base 1/1 | +| claude-code | ktune-04 | 109,109 | 107,304 | +1,805 | +1.68% | skill 1/1; base 1/1 | +| claude-code | ktune-05 | 295,709 | 332,266 | -36,557 | -11.00% | skill 1/1; base 1/1 | +| codex | All cases | 247,372 | 224,339 | +23,033 | +10.27% | skill 5/5; base 5/5 | +| codex | ktune-01 | 41,737 | 48,910 | -7,173 | -14.67% | skill 1/1; base 1/1 | +| codex | ktune-02 | 57,400 | 54,948 | +2,452 | +4.46% | skill 1/1; base 1/1 | +| codex | ktune-03 | 56,884 | 40,383 | +16,501 | +40.86% | skill 1/1; base 1/1 | +| codex | ktune-04 | 45,926 | 35,249 | +10,677 | +30.29% | skill 1/1; base 1/1 | +| codex | ktune-05 | 45,425 | 44,849 | +576 | +1.28% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,055,960 | 1,056,165 | -205 | -0.02% | skill 10/10; base 10/10 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 11 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-model-tuning/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/kumo-model-tuning/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/kumo-model-tuning/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/kumo-model-tuning/SKILL.md`) +- **MEDIUM** SCHEMA/author_missing: Author not specified in metadata (`skills/kumo-model-tuning/SKILL.md`) +- 6 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/kumo-model-tuning/skill-card.md b/skills/kumo-model-tuning/skill-card.md index e78088763..76c94201d 100644 --- a/skills/kumo-model-tuning/skill-card.md +++ b/skills/kumo-model-tuning/skill-card.md @@ -16,9 +16,9 @@ Global
## Requirements / Dependencies:
**Requires API Key or External Credential:** [Not Specified]
-**Credential Type(s):** [None identified]
+**Credential Type(s):** [None identified]
-Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
## Known Risks and Mitigations:
Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
@@ -26,6 +26,8 @@ Mitigation: Review and scan skill before deployment.
## Reference(s):
- [references/plans.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-tuning/references/plans.md)
+- [Kumo Training Diagnostics skill](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-training-diagnostics/SKILL.md)
+- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
@@ -35,8 +37,79 @@ Mitigation: Review and scan skill before deployment.
**Output Parameters:** [1D]
**Other Properties Related to Output:** [Suggesting a plan does not run its experiments.]
+## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.9`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 96.7% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 60.0% → 96.0% (+36.0 points) | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 47.5% → 90.0% (+42.5 points) | 51.7% → 80.8% (+29.1 points) | +| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 97.3% (-0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 91.6% → 92.3% (+0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 11 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | + ## Skill Version(s):
-Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+c7554e32 (source: git SHA, committed 2026-11-17); repository pyproject.toml version 0.1.0
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-model-tuning/skill.oms.sig b/skills/kumo-model-tuning/skill.oms.sig new file mode 100644 index 000000000..c75a8afbe --- /dev/null +++ b/skills/kumo-model-tuning/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC10dW5pbmciLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZGY3MjVmY2RkOGQ4ZGY1Mzc1ODBhYjdkOTcyMDQ1MWI4MGM0Yjg2Yjk3OWJmZTNkZWI4YWFlZDViMmIyNjcxZSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2JiYjNjM2YyNzYzNmVkNmNjOTZiOWIwMjE1OGZkNWYwY2U0MGIyNjI1OWYxMDYyN2UyMWYwMjdmY2VhZmM2ZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIxNjQwMWRiY2Y5OTM0ZTY1MTkyNDdhNDNiN2VkYTYxNzc5NjRiMTJmNzBiNjg2MTY0MWU0NDE2MGU2ZDhlN2FiIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiNThiNmNmMzIyNDI1M2VlZTVmOWMxZTAzYjFmODJlM2E5NWVlOGYxYzUyN2MyNWNiYjM1YzE3NTQ4OTBkZGZlYiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BsYW5zLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjNlYjQyYjBmODk1M2QxNzAwMjU1OGQwNWE4ODI4NTE5MzFkOGNiNTEwY2E4ZjdlNWM2MGViOWNmNjc5MDJjNGQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI4Zjk2NmQ3NmJhYTVhZGEyYWFiMjBlODdlZWZkMGYwNDVkZjdjOTQ3OWEyNTIyN2IyNzQ2YzRjMDMwZmU5NmVmIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCJQwv3fRQq8xPrzrwldJ0ix6jeqgvO96yPyz+goo9W6k8aKU6ISV5EE/XAC7klxNYCMQCSd0NyqWCO16VCWHQ8hMx9wSG8fW+IQwEPNMQPlcGXwDlDksUMQDUpg59I2481fe0=","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-training-diagnostics/BENCHMARK.md b/skills/kumo-training-diagnostics/BENCHMARK.md new file mode 100644 index 000000000..736a075a0 --- /dev/null +++ b/skills/kumo-training-diagnostics/BENCHMARK.md @@ -0,0 +1,155 @@ +# Skill Benchmark: kumo-training-diagnostics + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `kumo-training-diagnostics` +- Validation status: `passed` +- Evaluation date: 2026-11-18 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 5 evaluation tasks (4 positive, 1 negative) +- Dataset digest: `sha256:fb29a11af5d6eacb6479640b84fd298f7b83d35cd2c861416fde9b4a96dc23f0` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| kumo-training-diagnostics | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-training-diagnostics | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-training-diagnostics | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-training-diagnostics | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-training-diagnostics | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `kumo-training-diagnostics`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 88.9% — baseline ran, but no comparable score was available; uplift unavailable | 88.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | +| Correctness | 44.0% → 96.0% (+52.0 points) | 40.0% → 92.0% (+52.0 points) | +| Discoverability | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 48.2% → 58.5% (+10.3 points) | 50.2% → 63.2% (+13.0 points) | +| Efficiency | 95.1% — baseline ran, but no comparable score was available; uplift unavailable | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.4% → 96.5% (-1.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 92.2% → 91.8% (-0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 686,834 | 574,054 | +112,780 | +19.65% | skill 5/5; base 5/5 | +| claude-code | kdiag-01 | 197,505 | 107,243 | +90,262 | +84.17% | skill 1/1; base 1/1 | +| claude-code | kdiag-02 | 155,071 | 107,205 | +47,866 | +44.65% | skill 1/1; base 1/1 | +| claude-code | kdiag-03 | 73,175 | 144,125 | -70,950 | -49.23% | skill 1/1; base 1/1 | +| claude-code | kdiag-04 | 153,506 | 144,150 | +9,356 | +6.49% | skill 1/1; base 1/1 | +| claude-code | kdiag-05 | 107,577 | 71,331 | +36,246 | +50.81% | skill 1/1; base 1/1 | +| codex | All cases | 306,969 | 221,991 | +84,978 | +38.28% | skill 5/5; base 5/5 | +| codex | kdiag-01 | 61,333 | 40,873 | +20,460 | +50.06% | skill 1/1; base 1/1 | +| codex | kdiag-02 | 79,718 | 63,335 | +16,383 | +25.87% | skill 1/1; base 1/1 | +| codex | kdiag-03 | 61,395 | 42,697 | +18,698 | +43.79% | skill 1/1; base 1/1 | +| codex | kdiag-04 | 59,414 | 57,295 | +2,119 | +3.70% | skill 1/1; base 1/1 | +| codex | kdiag-05 | 45,109 | 17,791 | +27,318 | +153.55% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 993,803 | 796,045 | +197,758 | +24.84% | skill 10/10; base 10/10 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 12 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-training-diagnostics/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/kumo-training-diagnostics/SKILL.md`) +- **MEDIUM** QUALITY/quality_efficiency: Deeply nested references in diagnostics.md (`skills/kumo-training-diagnostics/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/kumo-training-diagnostics/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/kumo-training-diagnostics/SKILL.md`) +- 7 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/kumo-training-diagnostics/skill-card.md b/skills/kumo-training-diagnostics/skill-card.md index 381026a7b..7ead1f48f 100644 --- a/skills/kumo-training-diagnostics/skill-card.md +++ b/skills/kumo-training-diagnostics/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Evaluate an existing Kumo training run in a Python notebook using aggregate metrics, trial results, learning curves, refit evidence and training progress.
+Evaluate an existing Kumo training run in a Python notebook using aggregate metrics, trial results, learning curves, refit evidence, and training progress.
This skill is ready for commercial/non-commercial use.
@@ -9,34 +9,105 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill with coding agents to answer how a Kumo model performed, which trial was selected and what its learning curves and progress show, without retraining or tuning.
+Developers and data scientists use this skill with coding agents to answer how a Kumo model performed, which trial was selected, and what its learning curves and training progress show, without tuning, rerunning, or deploying a model.
### Deployment Geography for Use:
Global
## Requirements / Dependencies:
**Requires API Key or External Credential:** [Not Specified]
-**Credential Type(s):** [None identified]
+**Credential Type(s):** [None identified]
-Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
## Known Risks and Mitigations:
Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/diagnostics.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-training-diagnostics/references/diagnostics.md)
+- [references/diagnostics.md](references/diagnostics.md)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
## Skill Output:
-**Output Type(s):** [Code, Analysis]
+**Output Type(s):** [Analysis, Code]
**Output Format:** [Markdown assessment with inline Python code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Inspection only; does not tune, rerun or deploy a model.]
+**Other Properties Related to Output:** [Inspection only; does not tune, rerun, or deploy a model.]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated `k8s-sandbox` pod; each task was also run without the skill as a baseline. Evaluator version `1.6.9`, evaluated 2026-11-18.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+**Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 88.9% — baseline ran, but no comparable score was available; uplift unavailable | 88.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | +| Correctness | 44.0% → 96.0% (+52.0 points) | 40.0% → 92.0% (+52.0 points) | +| Discoverability | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 48.2% → 58.5% (+10.3 points) | 50.2% → 63.2% (+13.0 points) | +| Efficiency | 95.1% — baseline ran, but no comparable score was available; uplift unavailable | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.4% → 96.5% (-1.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 92.2% → 91.8% (-0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 12 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | ## Skill Version(s):
-Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+c7554e32 (source: git SHA, committed 2026-11-17)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-training-diagnostics/skill.oms.sig b/skills/kumo-training-diagnostics/skill.oms.sig new file mode 100644 index 000000000..6417a08aa --- /dev/null +++ b/skills/kumo-training-diagnostics/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby10cmFpbmluZy1kaWFnbm9zdGljcyIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJiZTVkNmZlNmVlMDQzODc5ZTAzMWY0Y2JlOTA5NzY0NmU5YmEzNWFkYzZjYjc0NmI3MGUwNGFmNDY2MjJkNmIxIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJmZmIzM2FhYmRjZWM5YmRhZDJhODVmODEwMGUxZTUzM2VjZDMyNTkyZTFhOWI5Yzg1OWNiNTcyMDU5OWYzMDE4IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjY0ZGQ4OGFlNjYwMmZiMWYyN2E0ZjQzYTRlNDJkNjA3ZjczZmQ5ZjE3YWE0ODgxNWEzOGNlOWY0MzQwZmVmNzYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICI4YzRjN2IzYTY5ZGQxNDcxYTAzYjI1NzNiN2JlZTE5NDExMjliN2E1MmM4OTk4ZWI4OTc3MjJkZjIxNTUyOTU0IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvZGlhZ25vc3RpY3MubWQiLAogICAgICAgICJkaWdlc3QiOiAiZTc1ZDMwZTU0NTUzMjJmODVjNTQwZDhiNDM3ZGIwYzAwNzYxZDUyNzU4YTUyZjNiOWE4YTc2ODAzNjFmY2UzMCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjkwMjE3MDllNWQ2NmYyMGZlMTM5MGVkOTM5ZTY3NmVjNWIyZjIwNTJiMDQ2Y2IzN2RmZDkwNmRiNzYzOGNjM2YiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDYqhjTNeHin6sfGPm8DpoxH4SAhTU/6p0Kgwe3FnUWxxDPZ+DZAXlT/PGvgqtSOSUCMECB5Kq6SNSlJQapSZ1A8Vyq07vSJ+3PQQtKgYujqftcqPS/wbLqnJiSGAgZhdZhzw==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumoai-reference/BENCHMARK.md b/skills/kumoai-reference/BENCHMARK.md new file mode 100644 index 000000000..b8bdcde49 --- /dev/null +++ b/skills/kumoai-reference/BENCHMARK.md @@ -0,0 +1,162 @@ +# Skill Benchmark: kumoai-reference + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `kumoai-reference` +- Validation status: `passed` +- Evaluation date: 2026-11-18 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 5 evaluation tasks (4 positive, 1 negative) +- Dataset digest: `sha256:70c6edab12642418c73b37fd93ff86ac13f4ce8f782eac415e5aa695a3c59505` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| kumoai-reference | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumoai-reference | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumoai-reference | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumoai-reference | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumoai-reference | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `kumoai-reference`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 94.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 80.0% → 100.0% (+20.0 points) | 90.0% → 100.0% (+10.0 points) | +| Correctness | 64.0% → 100.0% (+36.0 points) | 96.0% → 100.0% (+4.0 points) | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 50.2% → 80.7% (+30.5 points) | 68.9% → 91.6% (+22.7 points) | +| Efficiency | 87.7% — baseline ran, but no comparable score was available; uplift unavailable | 88.1% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 92.0% → 88.6% (-3.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 80.8% → 86.5% (+5.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 1,201,501 | 1,365,680 | -164,179 | -12.02% | skill 5/5; base 5/5 | +| claude-code | kref-01 | 121,639 | 35,557 | +86,082 | +242.10% | skill 1/1; base 1/1 | +| claude-code | kref-02 | 380,338 | 187,276 | +193,062 | +103.09% | skill 1/1; base 1/1 | +| claude-code | kref-03 | 233,155 | 549,777 | -316,622 | -57.59% | skill 1/1; base 1/1 | +| claude-code | kref-04 | 318,106 | 109,559 | +208,547 | +190.35% | skill 1/1; base 1/1 | +| claude-code | kref-05 | 148,263 | 483,511 | -335,248 | -69.34% | skill 1/1; base 1/1 | +| codex | All cases | 915,158 | 605,989 | +309,169 | +51.02% | skill 5/5; base 5/5 | +| codex | kref-01 | 64,938 | 46,310 | +18,628 | +40.22% | skill 1/1; base 1/1 | +| codex | kref-02 | 188,313 | 249,632 | -61,319 | -24.56% | skill 1/1; base 1/1 | +| codex | kref-03 | 135,073 | 75,281 | +59,792 | +79.43% | skill 1/1; base 1/1 | +| codex | kref-04 | 140,779 | 51,536 | +89,243 | +173.17% | skill 1/1; base 1/1 | +| codex | kref-05 | 386,055 | 183,230 | +202,825 | +110.69% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,116,659 | 1,971,669 | +144,990 | +7.35% | skill 10/10; base 10/10 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 2 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **HIGH** DUPLICATE/duplicate: Duplicate content found across references/graph_construction.md and references/kumo_finetuning_sdk.md: + "### Defining tables" in references/graph_construction.md (lines 106-143) + vs "## 4. Tables" in references/kumo_finetuning_sdk.md (lines 308-354) (`references/graph_construction.md:106`) +- **HIGH** DUPLICATE/duplicate: Duplicate content found across references/kumo_finetuning_sdk.md and references/pql_writing.md and references/task_journeys.md: + "## 6. Predictive query" in references/kumo_finetuning_sdk.md (lines 393-414) + vs "## Task type is inferred from query shape" in references/pql_writing.md (lines 202-220) + vs "## SDK usage" in references/pql_writing.md (lines 438-471) + vs "### Required pre-flight, before any training step" in references/pql_writing.md (lines 472-490) + vs "## Classification and regression" in references/task_journeys.md (lines 9-26) (`references/kumo_finetuning_sdk.md:393`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumoai-reference/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/kumoai-reference/SKILL.md`) +- **MEDIUM** QUALITY/quality_efficiency: Deeply nested references in outputs.md (`skills/kumoai-reference/SKILL.md`) +- 11 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/kumoai-reference/skill-card.md b/skills/kumoai-reference/skill-card.md index fdc400f7e..97495a8e1 100644 --- a/skills/kumoai-reference/skill-card.md +++ b/skills/kumoai-reference/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Reference for translating a validated ML request into KumoAI graph configuration, PQL (predictive query language) and Kumo SDK code.
+Reference for translating a validated ML request into KumoAI graph configuration, PQL (predictive query language) and Kumo Predict SDK code; normally loaded by the kumo-data-science skill after its intake checks pass.
This skill is ready for commercial/non-commercial use.
@@ -16,9 +16,9 @@ Global
## Requirements / Dependencies:
**Requires API Key or External Credential:** [Optional]
-**Credential Type(s):** [Cloud Credentials, OAuth Token, Other [Snowflake connections.toml entry]]
+**Credential Type(s):** [Cloud Credentials, OAuth Token, Other [Snowflake connections.toml entry or key pair]]
-Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
## Known Risks and Mitigations:
Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
@@ -31,6 +31,7 @@ Mitigation: Review and scan skill before deployment.
- [references/task_journeys.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/task_journeys.md)
- [references/outputs.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/outputs.md)
- [references/troubleshooting_signals.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/troubleshooting_signals.md)
+- [NVIDIA/kumo-engine#134 (graph visualization)](https://github.com/NVIDIA/kumo-engine/pull/134)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
@@ -38,10 +39,82 @@ Mitigation: Review and scan skill before deployment.
**Output Type(s):** [Code, Analysis]
**Output Format:** [Markdown with inline Python and PQL code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Generates code only; does not run jobs or interpret metrics.]
+**Other Properties Related to Output:** [Generates code only; does not run jobs, interpret metrics or repair runtime errors.]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.9`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 94.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 80.0% → 100.0% (+20.0 points) | 90.0% → 100.0% (+10.0 points) | +| Correctness | 64.0% → 100.0% (+36.0 points) | 96.0% → 100.0% (+4.0 points) | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 50.2% → 80.7% (+30.5 points) | 68.9% → 91.6% (+22.7 points) | +| Efficiency | 87.7% — baseline ran, but no comparable score was available; uplift unavailable | 88.1% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 92.0% → 88.6% (-3.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 80.8% → 86.5% (+5.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 2 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | + ## Skill Version(s):
-Unversioned; published from the main branch of NVIDIA/kumo-engine (source: repository; no skill version, CHANGELOG entry or release tag covers the skill files). The signed copy is identified by the skill.oms.sig file in this directory.
+c7554e32 (source: git SHA, committed 2026-11-17); repository pyproject.toml version 0.1.0
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumoai-reference/skill.oms.sig b/skills/kumoai-reference/skill.oms.sig new file mode 100644 index 000000000..690dd829f --- /dev/null +++ b/skills/kumoai-reference/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtb2FpLXJlZmVyZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJmODI4ZmY1OWM3ZjhhZGJiYTQ2ZmM0MTcxNTEzOTMwNTEyOWM5YmFlNzI5MThiZjhmYzNkODUxY2ExNzM1MzkyIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZjQwZDU1NDI4MzJlMTYyZDNjMzE3MjcyMDk3YzVlN2I4ZTEzYTJlNDIxZDgxYWE1OTE4OTUwZTk3ZTMxNDViOCIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMDQzMTYyYmFiYzc1NWFiYTEzNTgyNWFmM2NhMzc1ZDhmMGQ1MWQ5MDRjZGNhYmU2M2Y3MzlhNjNmODRjZDRhZiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlODBjYWNhY2NjNjA3YjIwMDFmYzE5MDQ0ZGNkNzEwOTJiNWEwNTQzYmY1NzYzMDJkZmRmMDgxMmY0NzhkZTZjIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOGU4OGUyM2IxN2EyMGUxN2E5Mjc4Njg3NDJmYzQwNTA2NGE4OGQxNDVmMmNiODgzNmE4NTJlMzQwMDVlNGJiYyIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9ncmFwaF9jb25zdHJ1Y3Rpb24ubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlMDQxYWY1ZTkyZmI2NmYyMjE1NWU0NTk0Mjk5NmFlZjM4ZWFjNGE0Y2E3ZDE0YjlkOWNhZWU4ZWFmNmI0OWI4IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2t1bW9fZmluZXR1bmluZ19zZGsubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJmMjlmM2QyZDJiY2E5NTUwZGU1MjViYjViYzYyZTU1ODc4ZTBmNTRiYTdmNWYxMDliMDJmYzk3OGJiYjg1ZWJhIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL291dHB1dHMubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI0NTc4N2RhY2I4ZTk3NzJhZTk1YjA1NzRmMjk3ODM0ODdlNGNjYzliNzBhZWUxYzkwNGQ4MGVkMmU4NWMxM2FmIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BxbF93cml0aW5nLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzA5YzIzYmExMGM3YWEwMDg5OTY0NjAwMGExNmVkYjMyYzg5ZDI5YzYzNTllNjE0ZGM5YWM2YTJlYTkyMGYwOCIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy90YXNrX2pvdXJuZXlzLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNzliNTU1ZGJmMmQ0N2MyOThlY2U2NGY0ZjVkODg0ZGU3N2Q0Zjc2YTY1M2JhOWUyMTgzMzBjM2YxYzE0N2ZiNSIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy90cm91Ymxlc2hvb3Rpbmdfc2lnbmFscy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjliYjFhYzFhYjE0N2ZmZDc0ZDIzMTBhMTRmODkyZTExNTAyMWMxOWM2MzIxNWVmNGY3MzIzZDMyMzg2MDIxNGEiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMH371JMAol5RWnITIr3SP2pAH+WgIt8OcP/IYb+/IwCXBCEkNfQkLADXYfRqfW/uswIwOIF3gjisxrvA3TJC5bH4gMutJpyOIl4DGnE5eL/9U/aeiFDx1a3XPrsiuXdATIhi","keyid":""}]}} \ No newline at end of file From 2857ed7dffbee769bd1e5e2bc46c4c01db520cb4 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Tue, 17 Nov 2026 21:52:11 -0600 Subject: [PATCH 25/35] agent-skills: keep the skill card NVSkills CI writes when it signs NVSkills CI rewrites each skill's skill-card.md in its signature commit, adding the evaluation results, so `--check` called the signed copy out of date. Once a skill directory holds a skill.oms.sig, its card now belongs to NVSkills CI like BENCHMARK.md: a refresh leaves it and `--check` does not compare it. The source card under agent-skills/nvskills/ seeds a skill's first card. Co-Authored-By: Claude Opus 5.5 --- agent-skills/README.md | 5 ++-- agent-skills/scripts/publish_nvskills.py | 33 ++++++++++++++++----- agent-skills/tests/test_publish_nvskills.py | 18 +++++++++++ 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/agent-skills/README.md b/agent-skills/README.md index 364e053a8..7a2da514d 100644 --- a/agent-skills/README.md +++ b/agent-skills/README.md @@ -151,7 +151,8 @@ and eval dataset: | Path, in each of the eight skill directories | Written by | |---|---| | everything else | `agent-skills/scripts/publish_nvskills.py`: the `generic` package | -| `skill-card.md`, `evals/` | copied from `agent-skills/nvskills//`, e.g. `agent-skills/nvskills/kumo-model-tuning/skill-card.md` | +| `evals/` | copied from `agent-skills/nvskills//`, e.g. `agent-skills/nvskills/kumo-model-tuning/evals/evals.json` | +| `skill-card.md` | first from `agent-skills/nvskills//`; NVSkills CI rewrites it when it signs, adding the evaluation results, and from then on a refresh leaves it alone. To change a signed card, edit `skills//skill-card.md` in your PR | | `BENCHMARK.md`, `skill.oms.sig` | NVSkills CI, in your PR; never by hand | Once `agent-skills/nvskills/` exists, the build fails unless every skill has both a card and an @@ -201,7 +202,7 @@ skill's own files document. Each set has one negative case (`expected_skill: nul entry per directory (`path: skills//`, `catalog_dir: `), all eight: NVIDIA/skills#885, a draft. Its `verify-source-onboarding` check verifies only the paths the file lists, and passes only once the signed copy is on `main` -- [ ] `request-nvskills-ci.yml` on `main` (NVIDIA/kumo-engine#371). `/nvskills-ci` is an +- [x] `request-nvskills-ci.yml` on `main` (NVIDIA/kumo-engine#371). `/nvskills-ci` is an `issue_comment` trigger, so it runs only from `main`'s copy - [ ] confirmation from the NVSkills CI maintainers that a skill may link to files in a sibling skill directory (`kumo-data-science/` reads `kumoai-reference/`). Each directory is signed diff --git a/agent-skills/scripts/publish_nvskills.py b/agent-skills/scripts/publish_nvskills.py index 31dcf3ce8..66b687966 100644 --- a/agent-skills/scripts/publish_nvskills.py +++ b/agent-skills/scripts/publish_nvskills.py @@ -31,7 +31,11 @@ * the catalog files that NVSkills requires and that we write: the skill's `skill-card.md` and `evals/*.json`. Their source is `agent-skills/nvskills//`, and they are copied in on every run. A catalog file must belong to a skill and may not replace a package file. Once that - source exists, the build fails unless every skill has both, so the sync never drops one; + source exists, the build fails unless every skill has both, so the sync never drops one. + The card is the exception to "every run": NVSkills CI rewrites it when it signs, adding the + evaluation results, so once a skill's directory holds a `skill.oms.sig` its card is NVSkills + CI's. A refresh then leaves it as it is and `--check` does not compare it. The source card + seeds a skill's first card; to change a signed one, edit `skills//skill-card.md`; * `BENCHMARK.md` and `skill.oms.sig`, which NVSkills CI writes into the pull request. They are never edited here, and the build fails if the package or the catalog files carry either name at any depth. A refresh leaves them in place at any depth, so they describe the previous @@ -203,14 +207,24 @@ def not_ready(catalog_files: Path = CATALOG_FILES) -> list[str]: return missing(build(Path(tmp), catalog_files)) -def files(root: Path) -> dict[str, bytes]: - """Map each file under `root` that we own to its bytes, with the build stamp normalized.""" +def signed_cards(root: Path) -> frozenset[str]: + """The cards under `root` that NVSkills CI rewrote when it signed their skill.""" + return frozenset(f'{skill}/{CARD}' for skill in skill_dirs(root) + if (root / skill / 'skill.oms.sig').is_file() + and (root / skill / CARD).is_file()) + + +def files(root: Path, kept: frozenset[str] = frozenset()) -> dict[str, bytes]: + """Map each file under `root` that we own to its bytes, with the build stamp normalized. + + `kept` names files NVSkills CI owns besides the signed ones: they are left out. + """ if not root.is_dir(): return {} out = {} for path in sorted(root.rglob('*')): rel = path.relative_to(root).as_posix() - if path.is_file() and path.name not in SIGNED: + if path.is_file() and path.name not in SIGNED and rel not in kept: data = path.read_bytes() if rel == f'{ROUTER}/SKILL.md': text = package_skill.MARKER.sub('SKILL-BUILD: x', data.decode('utf-8')) @@ -221,9 +235,10 @@ def files(root: Path) -> dict[str, bytes]: def stale(dest: Path = PUBLISHED, catalog_files: Path = CATALOG_FILES) -> list[str]: """Return what `dest` would gain, lose or change if it were refreshed now.""" + kept = signed_cards(dest) with tempfile.TemporaryDirectory() as tmp: - want = files(build(Path(tmp), catalog_files)) - have = files(dest) + want = files(build(Path(tmp), catalog_files), kept) + have = files(dest, kept) return ([f'+ {p}' for p in sorted(want.keys() - have.keys())] + [f'- {p}' for p in sorted(have.keys() - want.keys())] + [f'~ {p}' for p in sorted(want.keys() & have.keys()) if want[p] != have[p]]) @@ -234,16 +249,18 @@ def publish(dest: Path = PUBLISHED, catalog_files: Path = CATALOG_FILES) -> list changes = stale(dest, catalog_files) if not changes: return [] + kept = signed_cards(dest) with tempfile.TemporaryDirectory() as tmp: root = build(Path(tmp), catalog_files) if dest.is_dir(): for path in sorted(dest.rglob('*'), reverse=True): - if path.is_file() and path.name not in SIGNED: + rel = path.relative_to(dest).as_posix() + if path.is_file() and path.name not in SIGNED and rel not in kept: path.unlink() elif path.is_dir() and not any(path.iterdir()): path.rmdir() for path in sorted(root.rglob('*')): - if path.is_file(): + if path.is_file() and path.relative_to(root).as_posix() not in kept: target = dest / path.relative_to(root) target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(path, target) diff --git a/agent-skills/tests/test_publish_nvskills.py b/agent-skills/tests/test_publish_nvskills.py index f0ff99e70..2bb33c490 100644 --- a/agent-skills/tests/test_publish_nvskills.py +++ b/agent-skills/tests/test_publish_nvskills.py @@ -146,6 +146,24 @@ def test_publish_keeps_signed_files_and_drops_files_no_longer_shipped(tmp_path: assert path.read_text() == 'from NVSkills CI\n' +def test_a_signed_skill_keeps_the_card_nvskills_ci_wrote(tmp_path: Path, + catalog: Path) -> None: + """NVSkills CI rewrites a skill's card when it signs; the source card only seeds it.""" + dest = tmp_path / 'skills' + publish_nvskills.publish(dest, catalog) + signed, unsigned = dest / 'kumo-model-tuning', dest / 'kumoai-reference' + (signed / 'skill.oms.sig').write_text('from NVSkills CI\n') + for skill in (signed, unsigned): + (skill / 'skill-card.md').write_text('## Description\n\n## Evaluation Results\n') + + assert publish_nvskills.stale(dest, catalog) == ['~ kumoai-reference/skill-card.md'] + (dest / ROUTER / 'SKILL.md').write_text('edited by hand\n') + publish_nvskills.publish(dest, catalog) + assert (signed / 'skill-card.md').read_text() == '## Description\n\n## Evaluation Results\n' + assert (unsigned / 'skill-card.md').read_text() == '## Description\n' + assert publish_nvskills.stale(dest, catalog) == [] + + def test_check_ignores_the_build_stamp_but_not_content(tmp_path: Path, catalog: Path) -> None: dest = tmp_path / 'skills' publish_nvskills.publish(dest, catalog) From 5026ce470d56699da3e9ea2277460d6b578de4b4 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Tue, 17 Nov 2026 22:01:45 -0600 Subject: [PATCH 26/35] agent-skills: align a test signature for flake8 Co-Authored-By: Claude Opus 5.5 --- agent-skills/tests/test_publish_nvskills.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent-skills/tests/test_publish_nvskills.py b/agent-skills/tests/test_publish_nvskills.py index 2bb33c490..1c4bef53e 100644 --- a/agent-skills/tests/test_publish_nvskills.py +++ b/agent-skills/tests/test_publish_nvskills.py @@ -147,7 +147,7 @@ def test_publish_keeps_signed_files_and_drops_files_no_longer_shipped(tmp_path: def test_a_signed_skill_keeps_the_card_nvskills_ci_wrote(tmp_path: Path, - catalog: Path) -> None: + catalog: Path) -> None: """NVSkills CI rewrites a skill's card when it signs; the source card only seeds it.""" dest = tmp_path / 'skills' publish_nvskills.publish(dest, catalog) From 7cbda7af2df6e267a345070b36ae9e5b95c7eef1 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Wed, 18 Nov 2026 12:44:19 -0600 Subject: [PATCH 27/35] agent-skills: let sf_name keep an already-quoted Snowflake name The skill passes a lower-case database or schema to the connector with its double quotes (`schema_name='"my_schema"'`), and Kumo writes the table under that name. sf_name refused the quoted form, so the read-back raised before any SQL ran. A part that is a plain identifier or a complete double-quoted identifier now passes through unchanged; anything else is still refused. Co-Authored-By: Claude Opus 5.5 --- agent-skills/skills/hosts/generic/runtime.md | 10 ++++++---- skills/kumo-data-science/SKILL.md | 2 +- skills/kumo-data-science/hosts/generic/runtime.md | 10 ++++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/agent-skills/skills/hosts/generic/runtime.md b/agent-skills/skills/hosts/generic/runtime.md index 713485adf..18c05cae3 100644 --- a/agent-skills/skills/hosts/generic/runtime.md +++ b/agent-skills/skills/hosts/generic/runtime.md @@ -265,11 +265,13 @@ import re import snowflake.connector def sf_name(database, schema, table): - """DATABASE.SCHEMA."table": database and schema unquoted, so they must be plain identifiers; - the table quoted, with any double quote in it doubled.""" + """The table Kumo writes, as database.schema."table". Database and schema are written as + the connector was given them (§2): a plain identifier, which Snowflake upper-cases, or a + lower- or mixed-case name already in double quotes ('"my_schema"'), kept as it is. Anything + else is refused. The table is quoted, with any double quote in it doubled.""" for part in (database, schema): - if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_$]*', part or ''): - raise ValueError(f'not an unquoted Snowflake name: {part!r}; confirm it with the user') + if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_$]*|"(?:[^"]|"")+"', part or ''): + raise ValueError(f'not a Snowflake name: {part!r}; confirm it with the user') return f'{database}.{schema}."' + table.replace('"', '""') + '"' with snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index bebf711ae..9f75f4136 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-17-53c38923-dirty`** +**`SKILL-BUILD: 2026-11-17-5026ce47-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/hosts/generic/runtime.md b/skills/kumo-data-science/hosts/generic/runtime.md index 713485adf..18c05cae3 100644 --- a/skills/kumo-data-science/hosts/generic/runtime.md +++ b/skills/kumo-data-science/hosts/generic/runtime.md @@ -265,11 +265,13 @@ import re import snowflake.connector def sf_name(database, schema, table): - """DATABASE.SCHEMA."table": database and schema unquoted, so they must be plain identifiers; - the table quoted, with any double quote in it doubled.""" + """The table Kumo writes, as database.schema."table". Database and schema are written as + the connector was given them (§2): a plain identifier, which Snowflake upper-cases, or a + lower- or mixed-case name already in double quotes ('"my_schema"'), kept as it is. Anything + else is refused. The table is quoted, with any double quote in it doubled.""" for part in (database, schema): - if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_$]*', part or ''): - raise ValueError(f'not an unquoted Snowflake name: {part!r}; confirm it with the user') + if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_$]*|"(?:[^"]|"")+"', part or ''): + raise ValueError(f'not a Snowflake name: {part!r}; confirm it with the user') return f'{database}.{schema}."' + table.replace('"', '""') + '"' with snowflake.connector.connect(connection_name=SNOWFLAKE_CONNECTION, From f22b610d4b68f6c4c3d25e9f08702ced1d865401 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Wed, 18 Nov 2026 19:59:09 +0000 Subject: [PATCH 28/35] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/kumo-custom-populations/BENCHMARK.md | 52 +++++------ skills/kumo-custom-populations/skill-card.md | 58 +++++++------ skills/kumo-custom-populations/skill.oms.sig | 2 +- skills/kumo-data-science/BENCHMARK.md | 70 +++++++-------- skills/kumo-data-science/skill-card.md | 86 +++++++++---------- skills/kumo-data-science/skill.oms.sig | 2 +- skills/kumo-graph-visualization/BENCHMARK.md | 52 +++++------ skills/kumo-graph-visualization/skill-card.md | 18 ++-- skills/kumo-graph-visualization/skill.oms.sig | 2 +- skills/kumo-model-explanations/BENCHMARK.md | 54 ++++++------ skills/kumo-model-explanations/skill-card.md | 46 ++++------ skills/kumo-model-explanations/skill.oms.sig | 2 +- skills/kumo-model-persistence/BENCHMARK.md | 56 ++++++------ skills/kumo-model-persistence/skill-card.md | 26 +++--- skills/kumo-model-persistence/skill.oms.sig | 2 +- skills/kumo-model-tuning/BENCHMARK.md | 52 +++++------ skills/kumo-model-tuning/skill-card.md | 67 ++++++--------- skills/kumo-model-tuning/skill.oms.sig | 2 +- skills/kumo-training-diagnostics/BENCHMARK.md | 52 +++++------ .../kumo-training-diagnostics/skill-card.md | 18 ++-- .../kumo-training-diagnostics/skill.oms.sig | 2 +- skills/kumoai-reference/BENCHMARK.md | 64 +++++++------- skills/kumoai-reference/skill-card.md | 29 +++---- skills/kumoai-reference/skill.oms.sig | 2 +- 24 files changed, 394 insertions(+), 422 deletions(-) diff --git a/skills/kumo-custom-populations/BENCHMARK.md b/skills/kumo-custom-populations/BENCHMARK.md index 387d62dd4..f049f04b7 100644 --- a/skills/kumo-custom-populations/BENCHMARK.md +++ b/skills/kumo-custom-populations/BENCHMARK.md @@ -11,7 +11,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-custom-populations` - Validation status: `passed` - Evaluation date: 2026-11-18 -- Evaluator version: `1.6.9` +- Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 5 evaluation tasks (4 positive, 1 negative) - Dataset digest: `sha256:ed3c0e36c21c14588ddb79e7ed42113b76bddfa7ad9f90f3e27392b0fffdf2f2` (skill-evaluator-dataset-snapshot/1) @@ -28,11 +28,11 @@ SkillSpector provider, model, analyzer revision, and determinism evidence retain | Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | | --- | --- | --- | --- | --- | --- | --- | -| kumo-custom-populations | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-custom-populations | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-custom-populations | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-custom-populations | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-custom-populations | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-custom-populations | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-custom-populations | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-custom-populations | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-custom-populations | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-custom-populations | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | - Controls for `kumo-custom-populations`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.9% — baseline ran, but no comparable score was available; uplift unavailable | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 95.1% — baseline ran, but no comparable score was available; uplift unavailable | 90.2% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 56.0% → 100.0% (+44.0 points) | 64.0% → 92.0% (+28.0 points) | -| Discoverability | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 46.0% → 70.0% (+24.0 points) | 42.0% → 84.0% (+42.0 points) | -| Efficiency | 98.2% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 52.0% → 100.0% (+48.0 points) | 68.0% → 96.0% (+28.0 points) | +| Discoverability | 97.5% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 44.0% → 80.0% (+36.0 points) | 42.0% → 64.0% (+22.0 points) | +| Efficiency | 98.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 92.6% → 96.3% (+3.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 96.2% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 88.2% → 92.2% (+4.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.0% → 92.2% (+3.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 610,996 | 614,031 | -3,035 | -0.49% | skill 5/5; base 5/5 | -| claude-code | kpop-01 | 154,881 | 145,240 | +9,641 | +6.64% | skill 1/1; base 1/1 | -| claude-code | kpop-02 | 153,305 | 143,539 | +9,766 | +6.80% | skill 1/1; base 1/1 | -| claude-code | kpop-03 | 112,504 | 106,841 | +5,663 | +5.30% | skill 1/1; base 1/1 | -| claude-code | kpop-04 | 154,625 | 182,680 | -28,055 | -15.36% | skill 1/1; base 1/1 | -| claude-code | kpop-05 | 35,681 | 35,731 | -50 | -0.14% | skill 1/1; base 1/1 | -| codex | All cases | 192,572 | 257,130 | -64,558 | -25.11% | skill 5/5; base 5/5 | -| codex | kpop-01 | 43,740 | 66,096 | -22,356 | -33.82% | skill 1/1; base 1/1 | -| codex | kpop-02 | 43,769 | 45,971 | -2,202 | -4.79% | skill 1/1; base 1/1 | -| codex | kpop-03 | 43,649 | 45,724 | -2,075 | -4.54% | skill 1/1; base 1/1 | -| codex | kpop-04 | 43,767 | 81,593 | -37,826 | -46.36% | skill 1/1; base 1/1 | -| codex | kpop-05 | 17,647 | 17,746 | -99 | -0.56% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 803,568 | 871,161 | -67,593 | -7.76% | skill 10/10; base 10/10 | +| claude-code | All cases | 694,555 | 651,222 | +43,333 | +6.65% | skill 5/5; base 5/5 | +| claude-code | kpop-01 | 153,733 | 146,041 | +7,692 | +5.27% | skill 1/1; base 1/1 | +| claude-code | kpop-02 | 195,226 | 144,634 | +50,592 | +34.98% | skill 1/1; base 1/1 | +| claude-code | kpop-03 | 154,848 | 106,828 | +48,020 | +44.95% | skill 1/1; base 1/1 | +| claude-code | kpop-04 | 155,043 | 218,081 | -63,038 | -28.91% | skill 1/1; base 1/1 | +| claude-code | kpop-05 | 35,705 | 35,638 | +67 | +0.19% | skill 1/1; base 1/1 | +| codex | All cases | 209,249 | 276,533 | -67,284 | -24.33% | skill 5/5; base 5/5 | +| codex | kpop-01 | 43,763 | 61,911 | -18,148 | -29.31% | skill 1/1; base 1/1 | +| codex | kpop-02 | 60,254 | 60,332 | -78 | -0.13% | skill 1/1; base 1/1 | +| codex | kpop-03 | 43,684 | 55,934 | -12,250 | -21.90% | skill 1/1; base 1/1 | +| codex | kpop-04 | 43,771 | 80,750 | -36,979 | -45.79% | skill 1/1; base 1/1 | +| codex | kpop-05 | 17,777 | 17,606 | +171 | +0.97% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 903,804 | 927,755 | -23,951 | -2.58% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-custom-populations/skill-card.md b/skills/kumo-custom-populations/skill-card.md index 10614fc17..cf57d4da4 100644 --- a/skills/kumo-custom-populations/skill-card.md +++ b/skills/kumo-custom-populations/skill-card.md @@ -9,14 +9,14 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill with coding agents when generated Kumo training or prediction tables do not represent the rows they need, for example curated labels, manual splits, sample weights or an exact scoring population.
+Developers and data scientists use this skill to train Kumo models on caller-supplied labels, weights and splits, or to score exact entity/anchor rows from Snowflake source tables, when generated populations do not represent the requested rows.
### Deployment Geography for Use:
Global
## Requirements / Dependencies:
-**Requires API Key or External Credential:** [Yes]
-**Credential Type(s):** [Other [Snowflake connection or Snowpark session of the user's notebook]]
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
@@ -25,15 +25,15 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/populations.md](references/populations.md)
-- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+- [Supplied Snowflake population contract](references/populations.md)
+- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
## Skill Output:
-**Output Type(s):** [Code, Analysis]
-**Output Format:** [Markdown with inline Python and SQL code blocks]
+**Output Type(s):** [Analysis, Code]
+**Output Format:** [Markdown with inline Python code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Reads source tables only through read-only queries; never writes back to them.]
+**Other Properties Related to Output:** [Presents a plan and stops for user confirmation before generating training or scoring code.]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -42,7 +42,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod (`k8s-sandbox`); each task was also run without the skill as a baseline. Evaluator version `1.6.9`, evaluated 2026-11-18.
+5 evaluation tasks (4 positive, 1 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`); evaluator version `1.6.10`; evaluated 2026-11-18.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -53,13 +53,13 @@ Reported benchmark dimensions:
- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Accuracy: final-answer correctness against the reference answer.
-- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
-- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
-- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
@@ -74,12 +74,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.9% — baseline ran, but no comparable score was available; uplift unavailable | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 95.1% — baseline ran, but no comparable score was available; uplift unavailable | 90.2% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 56.0% → 100.0% (+44.0 points) | 64.0% → 92.0% (+28.0 points) | -| Discoverability | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 46.0% → 70.0% (+24.0 points) | 42.0% → 84.0% (+42.0 points) | -| Efficiency | 98.2% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 52.0% → 100.0% (+48.0 points) | 68.0% → 96.0% (+28.0 points) | +| Discoverability | 97.5% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 44.0% → 80.0% (+36.0 points) | 42.0% → 64.0% (+22.0 points) | +| Efficiency | 98.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -87,19 +87,27 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. -#### Comparison details +### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 92.6% → 96.3% (+3.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 96.2% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 88.2% → 92.2% (+4.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.0% → 92.2% (+3.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 11 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | ## Skill Version(s):
-c7554e32 (source: git SHA, committed 2026-11-17)
+0.1.0 (source: pyproject.toml)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-custom-populations/skill.oms.sig b/skills/kumo-custom-populations/skill.oms.sig index cbfb38cd5..1b319007e 100644 --- a/skills/kumo-custom-populations/skill.oms.sig +++ b/skills/kumo-custom-populations/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1jdXN0b20tcG9wdWxhdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZjRjNGQ4Yzk3ZjA5MjQ2YzAwNGE2NmRjYzhmNjQ2MGQ4MTJkM2NmMTNiZTUzNGQwYWI1NGFkNzM4YzRkMzg4ZSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMTg1OTJiMjZlYTFiMDM2MjZiMjNmYTc5ZTc1YWMzNGEzNDdlYzcxYzExODk1ZTA0MzQwMTc0NDEyNjQ5YTBjMiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2MjI4MmZkMzY0MmI0MzMzNjY1NjJiYTU5YTUzYmE1YjkzMDI1N2U3M2UzYWQ1OGI5NmQ2OGMyNzAwZjM5NzU4IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzA1ZjU2YzBkNzJmMzViZWE5ZDIxMzBlNTYyZTUwZjk5YTEzYjkxMTZkOWY5ZDRkM2Q3ZTVmOTdkYWM5ZDdkOCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcG9wdWxhdGlvbnMubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjdmMjJmYjQzN2UwY2YwMGQxYzBjZWVlNjJhMGJiN2VmMGFmMzU0OGQzNDIyOWVmZmQxYzA1NWNiOWUyYWQyN2EiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIwZGQ5YWVlZjVhYjc4NGRlZmI4NGNhOTMxMTZkMzE1ZjliMWFlYzQzZTNhZDRiODIwNjYyOTlhMjJjZjI1NTNkIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQC8CrAmiDzpN+Dx+u+hxUj9EkPOZu5zmP0KIJfsljavnOblD1q9LlgMRJhlbbGQENkCMAC6JIPreLCihw8WDQIemS4AS4RHjmBdgRnvmAxQIUgPMRcnz7Cpmn4i68XHD3VFbw==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1jdXN0b20tcG9wdWxhdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNTcyMWNkNDBjZDJhYjZjYmM3OTc5Y2ZiNDQwNzgxZDE1MDcyMGU4ZTQ3OGY5ZDgzZjIwMjMwNDUzZWRlZjIyMiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYThjYzk1MmVhNjc2ZWUyNjk2ODRiZWZjMjExYjMwMzMzZjdmN2QyOTU0ZWZhYTlmNTg3Nzc0ODdkN2QxOWY5ZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNjIyODJmZDM2NDJiNDMzMzY2NTYyYmE1OWE1M2JhNWI5MzAyNTdlNzNlM2FkNThiOTZkNjhjMjcwMGYzOTc1OCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI3MDVmNTZjMGQ3MmYzNWJlYTlkMjEzMGU1NjJlNTBmOTlhMTNiOTExNmQ5ZjlkNGQzZDdlNWY5N2RhYzlkN2Q4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiN2YyMmZiNDM3ZTBjZjAwZDFjMGNlZWU2MmEwYmI3ZWYwYWYzNTQ4ZDM0MjI5ZWZmZDFjMDU1Y2I5ZTJhZDI3YSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcG9wdWxhdGlvbnMubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImM0NDk2MGNmMjI0YjY0ODNiMzQwMTU5YWNjM2YwMWUxYzdkNzM4ZjRiNjEwMTgyZDMwOWM2Zjc0ZjRmMWJhYjEiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIKICAgICAgXQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCHGw/myeqEVuOCOjZhu4CyeMGEJMrcBlIUEKygqBOlqs7lkEazg/emr5ubSBK1QtECMAkrD0w3DeZkq+X+ivOOHETQE7cOjbvvITzDbbGV5UuAjs/B8YhFI/Wq0/3lSVE1eQ==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-data-science/BENCHMARK.md b/skills/kumo-data-science/BENCHMARK.md index 230d43550..e50d93787 100644 --- a/skills/kumo-data-science/BENCHMARK.md +++ b/skills/kumo-data-science/BENCHMARK.md @@ -11,7 +11,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-data-science` - Validation status: `passed` - Evaluation date: 2026-11-18 -- Evaluator version: `1.6.9` +- Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 8 evaluation tasks (7 positive, 1 negative) - Dataset digest: `sha256:f72dc6983fe9c6ff4f46b5d40bebd4ceb50356c04e83ebc88dc2cd383486d627` (skill-evaluator-dataset-snapshot/1) @@ -28,11 +28,11 @@ SkillSpector provider, model, analyzer revision, and determinism evidence retain | Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | | --- | --- | --- | --- | --- | --- | --- | -| kumo-data-science | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-data-science | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-data-science | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-data-science | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-data-science | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-data-science | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-data-science | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-data-science | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-data-science | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-data-science | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | - Controls for `kumo-data-science`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 99.1% — baseline ran, but no comparable score was available; uplift unavailable | 94.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 93.8% → 100.0% (+6.2 points) | -| Correctness | 60.0% → 100.0% (+40.0 points) | 47.5% → 100.0% (+52.5 points) | -| Discoverability | 99.3% — baseline ran, but no comparable score was available; uplift unavailable | 90.7% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 39.3% → 100.0% (+60.7 points) | 38.8% → 85.9% (+47.1 points) | -| Efficiency | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 91.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 87.5% → 100.0% (+12.5 points) | +| Correctness | 52.5% → 100.0% (+47.5 points) | 47.5% → 92.5% (+45.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 42.6% → 100.0% (+57.4 points) | 38.8% → 80.4% (+41.6 points) | +| Efficiency | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.1% → 92.5% (-4.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.0% → 92.4% (-4.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 83.3% → 90.4% (+7.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 83.2% → 90.3% (+7.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,25 +80,25 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,561,766 | 1,074,705 | +487,061 | +45.32% | skill 8/8; base 8/8 | -| claude-code | kds-01 | 227,358 | 143,260 | +84,098 | +58.70% | skill 1/1; base 1/1 | -| claude-code | kds-02 | 184,081 | 192,796 | -8,715 | -4.52% | skill 1/1; base 1/1 | -| claude-code | kds-03 | 336,605 | 273,297 | +63,308 | +23.16% | skill 1/1; base 1/1 | -| claude-code | kds-04 | 219,379 | 108,298 | +111,081 | +102.57% | skill 1/1; base 1/1 | -| claude-code | kds-05 | 172,823 | 106,455 | +66,368 | +62.34% | skill 1/1; base 1/1 | -| claude-code | kds-06 | 216,411 | 107,743 | +108,668 | +100.86% | skill 1/1; base 1/1 | -| claude-code | kds-07 | 169,197 | 107,191 | +62,006 | +57.85% | skill 1/1; base 1/1 | -| claude-code | kds-08 | 35,912 | 35,665 | +247 | +0.69% | skill 1/1; base 1/1 | -| codex | All cases | 617,755 | 3,018,694 | -2,400,939 | -79.54% | skill 8/8; base 8/8 | -| codex | kds-01 | 71,880 | 44,978 | +26,902 | +59.81% | skill 1/1; base 1/1 | -| codex | kds-02 | 97,244 | 1,714,885 | -1,617,641 | -94.33% | skill 1/1; base 1/1 | -| codex | kds-03 | 96,671 | 1,061,264 | -964,593 | -90.89% | skill 1/1; base 1/1 | -| codex | kds-04 | 117,090 | 41,458 | +75,632 | +182.43% | skill 1/1; base 1/1 | -| codex | kds-05 | 67,909 | 43,899 | +24,010 | +54.69% | skill 1/1; base 1/1 | -| codex | kds-06 | 84,028 | 41,072 | +42,956 | +104.59% | skill 1/1; base 1/1 | -| codex | kds-07 | 69,274 | 57,676 | +11,598 | +20.11% | skill 1/1; base 1/1 | -| codex | kds-08 | 13,659 | 13,462 | +197 | +1.46% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 2,179,521 | 4,093,399 | -1,913,878 | -46.76% | skill 16/16; base 16/16 | +| claude-code | All cases | 1,593,311 | 932,554 | +660,757 | +70.85% | skill 8/8; base 8/8 | +| claude-code | kds-01 | 217,339 | 143,640 | +73,699 | +51.31% | skill 1/1; base 1/1 | +| claude-code | kds-02 | 184,163 | 166,351 | +17,812 | +10.71% | skill 1/1; base 1/1 | +| claude-code | kds-03 | 237,291 | 193,342 | +43,949 | +22.73% | skill 1/1; base 1/1 | +| claude-code | kds-04 | 316,252 | 108,091 | +208,161 | +192.58% | skill 1/1; base 1/1 | +| claude-code | kds-05 | 216,613 | 70,836 | +145,777 | +205.80% | skill 1/1; base 1/1 | +| claude-code | kds-06 | 216,619 | 107,275 | +109,344 | +101.93% | skill 1/1; base 1/1 | +| claude-code | kds-07 | 169,166 | 107,391 | +61,775 | +57.52% | skill 1/1; base 1/1 | +| claude-code | kds-08 | 35,868 | 35,628 | +240 | +0.67% | skill 1/1; base 1/1 | +| codex | All cases | 603,483 | 3,473,502 | -2,870,019 | -82.63% | skill 8/8; base 8/8 | +| codex | kds-01 | 71,810 | 45,041 | +26,769 | +59.43% | skill 1/1; base 1/1 | +| codex | kds-02 | 72,570 | 2,079,924 | -2,007,354 | -96.51% | skill 1/1; base 1/1 | +| codex | kds-03 | 119,713 | 1,151,989 | -1,032,276 | -89.61% | skill 1/1; base 1/1 | +| codex | kds-04 | 71,905 | 41,272 | +30,633 | +74.22% | skill 1/1; base 1/1 | +| codex | kds-05 | 86,631 | 26,266 | +60,365 | +229.82% | skill 1/1; base 1/1 | +| codex | kds-06 | 48,715 | 40,986 | +7,729 | +18.86% | skill 1/1; base 1/1 | +| codex | kds-07 | 118,454 | 74,560 | +43,894 | +58.87% | skill 1/1; base 1/1 | +| codex | kds-08 | 13,685 | 13,464 | +221 | +1.64% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,196,794 | 4,406,056 | -2,209,262 | -50.14% | skill 16/16; base 16/16 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -106,7 +106,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 27 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 26 finding(s) | | Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 4 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 8 task(s) | @@ -130,7 +130,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c vs "### `batch_inference` — required fields (ask if missing)" in accept/batch_inference.md (lines 30-42) vs "### `batch_inference` — resolved, defaulted, or inferred (never ask)" in accept/batch_inference.md (lines 43-54) (`accept/batch_inference.md:7`) - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-data-science/SKILL.md`) -- 26 additional finding(s) are available in the full evaluation artifacts. +- 25 additional finding(s) are available in the full evaluation artifacts. diff --git a/skills/kumo-data-science/skill-card.md b/skills/kumo-data-science/skill-card.md index 50954d964..eef55ab15 100644 --- a/skills/kumo-data-science/skill-card.md +++ b/skills/kumo-data-science/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Turns a plain-language request for a KumoAI predictive model on relational data in a warehouse, an object store or local files into a confirmed task and graph through a multi-turn intake, generates one notebook or script that trains and predicts with the local Kumo SDK (running it only when asked), and routes graph inspection, explanation, diagnostics, persistence, custom-population and tuning requests to their own Kumo skills.
+Routes KumoAI graph and predictive-modeling requests on relational data in a warehouse, an object store or local files, and for a new model confirms the data, outcome, time window, inferred tables and task through a gated intake before generating one train-then-predict notebook or script, which it runs only when asked.
This skill is ready for commercial/non-commercial use.
@@ -9,14 +9,14 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill with coding agents to build Kumo models for churn, propensity, fraud, recommendation, forecasting and link prediction on relational data in Snowflake, BigQuery, Databricks, S3, GCS or local files: the agent checks the request, proposes the graph and task for confirmation and writes the training-and-prediction notebook, or helps create and debug the jobs that run it.
+Developers, data scientists and ML engineers use this skill to turn plain-language predictive requests (churn, propensity, fraud, recommendation, forecasting, link prediction) on relational data into a confirmed KumoAI task and graph and one runnable notebook or script, and to route graph inspection, explanation, diagnostics, persistence, custom-population, tuning and job requests to their dedicated skills.
### Deployment Geography for Use:
Global
## Requirements / Dependencies:
-**Requires API Key or External Credential:** [Optional]
-**Credential Type(s):** [Cloud Credentials, Service Account, Other [Snowflake connections.toml entry]]
+**Requires API Key or External Credential:** [Yes]
+**Credential Type(s):** [Cloud Credentials]
Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
@@ -25,23 +25,23 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [accept/intake_contract.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/intake_contract.md)
-- [accept/preflight.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/preflight.md)
-- [accept/batch_inference.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/batch_inference.md)
-- [accept/semantic_view.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/accept/semantic_view.md)
-- [validate/atomic_question_check.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/validate/atomic_question_check.md)
-- [hosts/generic/adapter.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/hosts/generic/adapter.md)
-- [hosts/generic/runtime.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-data-science/hosts/generic/runtime.md)
-- [NVIDIA/kumo-engine#134 (graph visualization)](https://github.com/NVIDIA/kumo-engine/pull/134)
+- [Intake contract](accept/intake_contract.md)
+- [Preflight and graph gate](accept/preflight.md)
+- [Batch inference](accept/batch_inference.md)
+- [Semantic views](accept/semantic_view.md)
+- [Atomic question check](validate/atomic_question_check.md)
+- [Generic host adapter](hosts/generic/adapter.md)
+- [Generic host runtime](hosts/generic/runtime.md)
- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
-- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+- [kumo-engine on PyPI](https://pypi.org/project/kumo-engine/)
+- [NVIDIA/kumo-engine#134 (graph visualization)](https://github.com/NVIDIA/kumo-engine/pull/134)
## Skill Output:
-**Output Type(s):** [Code, Analysis]
-**Output Format:** [Markdown replies (ask-backs, graph proposal, task restatement) and one Jupyter notebook or Python script]
+**Output Type(s):** [Analysis, Code, Files]
+**Output Format:** [Markdown replies (graph and task proposals, ask-backs) and one generated Jupyter notebook or Python script using the kumo.engine API]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Generates code only; runs it only when the user asks. Writes predictions only to a destination the user confirmed.]
+**Other Properties Related to Output:** [Generates code only until the user confirms the graph and task; runs the artifact only on explicit request]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -50,7 +50,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-8 evaluation tasks (7 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.9`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+8 evaluation tasks (7 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox pod (`k8s-sandbox`), compared against a no-skill baseline. Evaluator version `1.6.10`; evaluated 2026-11-18.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -61,61 +61,53 @@ Reported benchmark dimensions:
- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Accuracy: final-answer correctness against the reference answer.
-- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
-- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
-- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
-> ✅ **Overall verdict: PASS — Recommended for publication** - -### Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. - ### Results at a Glance | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 99.1% — baseline ran, but no comparable score was available; uplift unavailable | 94.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 93.8% → 100.0% (+6.2 points) | -| Correctness | 60.0% → 100.0% (+40.0 points) | 47.5% → 100.0% (+52.5 points) | -| Discoverability | 99.3% — baseline ran, but no comparable score was available; uplift unavailable | 90.7% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 39.3% → 100.0% (+60.7 points) | 38.8% → 85.9% (+47.1 points) | -| Efficiency | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 91.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 87.5% → 100.0% (+12.5 points) | +| Correctness | 52.5% → 100.0% (+47.5 points) | 47.5% → 92.5% (+45.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 42.6% → 100.0% (+57.4 points) | 38.8% → 80.4% (+41.6 points) | +| Efficiency | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | -**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. - -Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. - -A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; uplift is derived from the displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. #### Comparison details -Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. - | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.1% → 92.5% (-4.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.0% → 92.4% (-4.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 83.3% → 90.4% (+7.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 83.2% → 90.3% (+7.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 27 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 26 finding(s) | | Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 4 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 8 task(s) | +**Overall verdict: PASS — Recommended for publication** + +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. + ## Skill Version(s):
-c7554e32 (source: git SHA, committed 2026-11-17); repository pyproject.toml version 0.1.0; SKILL.md build marker SKILL-BUILD: 2026-11-17-53c38923-dirty
+0.1.0 (source: pyproject.toml); skill build marker 2026-11-17-5026ce47-dirty (source: SKILL.md); git 7cbda7af (committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-data-science/skill.oms.sig b/skills/kumo-data-science/skill.oms.sig index 8e25c5ce5..85af9b7a4 100644 --- a/skills/kumo-data-science/skill.oms.sig +++ b/skills/kumo-data-science/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1kYXRhLXNjaWVuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYzQ5NmE3MGI1Zjk1NTE4ZjFhMTI0MTI0ODA0NmMxMmM4MDkxNjVlYWY2ZjcyNzQ2Yzk1NTkwMWY5N2NkOTkzOSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZjQzN2VjOWFhZjJiMDIzZGRhYWU0MzM0M2MzNmQzNDI3NWE0ZDNiZDgxNmIwNjUwYWZjOTZjZDI2ZjcxZjE3OSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMjM0YjAxOTU3MGIyMGJmNGJkZmQ3ZGU2YjNlYzU4ODJiYmUwMTg2Y2FiYjVmZDY4NDEzOTRmOTg5MzRlZDZjMCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIyZDc1YWJhMGZhNGZlMTliODg1NjUzMjgwZTFhNTJkMTNiMDdhMGUxMzNkYzQzNGQxYjJiZDdhYjBmOWY2YmI4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L2JhdGNoX2luZmVyZW5jZS5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZmVhN2JiZDg2NWI4MzY2NDExMDdmNzQwMzQ4MjZiMWY1NGExNGQxNGYzOGIxN2VhYzBmZGRhODA4OTQxMjJkMSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFjY2VwdC9pbnRha2VfY29udHJhY3QubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImQzYjRjOTA3NDE1ZWE0MzYwZTA5YTRhYTIyZTljYThlNjhiZDIzZjhiNGM0N2NjZjlkMWI5NWQ4YWE2MWJlOWYiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJhY2NlcHQvcHJlZmxpZ2h0Lm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJmOWYyZGY5Y2FhNGQ1NjllNDNlNGQzNGY3MDk5YjIxNTExZmFjMWFiMzllZDYyMjczMjQ4ZmI2MTFiOWNiZDNlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L3NlbWFudGljX3ZpZXcubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjY2NDA0ZDljYjEzZDAwOTNiNjcwMWRmZDRiMTkzMzcxMDNkOGQzYjg2MDc1ZTExMTM5MzZkM2I0NGU5OWNjYTkiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJjNmEzNjg0NTRmYTlhZGVlYmEwYmRkYTk4YmQzZDIxNGYyYjkyYTVmMTI4MzUyNzQ1YzNkMjJlYjkwN2I1YjZiIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiaG9zdHMvZ2VuZXJpYy9hZGFwdGVyLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIxM2I3Y2UwMmU3MjU1YzY3ODNiODVlYWJhMmVjYThiMWY5ODU3YWUxM2FiNDQ3ZWI1NDdmZjdhMWExM2YyMjNjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiaG9zdHMvZ2VuZXJpYy9ydW50aW1lLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJiM2Q0YmUwYjNiNTQzN2E0ZTc4MTg1NTNkNmJkNzhlOGExYTBlODgyNDc5NTYxYmUzYWMxMGIwODZjODc2M2Y3IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNjUwNmU2NjFiYzExYzI1ZWI2NmE3YjdjMzVhMzQwMzEzOTU3ZDMyNGJhMzg2N2U5NTA4NmIxMTRlOTJkMjA0YSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2F0b21pY19xdWVzdGlvbl9jaGVjay5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiN2JkMzkzNGUxYjljMjJhNGMzOGIzN2JjYjhjNTk3MjkyZGRlZTgyMjA2MWQ5ZmVlZWU2NTdiZDkwNTAzNWNkNCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2V4YW1wbGVzL2JhZF9yZXF1ZXN0Lm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI4OTJkMGZiZWI0MmE4MDJkMzM2ZmZiMjMwOWQ1ZTk5OWIzOGNmYjQ4N2QzNjljMzg5Mjk0OWYzY2MxYmVkOGU2IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAidmFsaWRhdGUvZXhhbXBsZXMvZ29vZF9yZXF1ZXN0Lm1kIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQC0C2zqrFD3/TElDbjwOz15rlNcHqRk/J8ka9DCxR/Jg0UhKcOvpTKUNNMpqO0F+FMCMGQmSWA3F7fo3UcG3u1osYLviYg2WoHqos9oIjjx2QrYFRqJEygorvRo6TwZwiIcbw==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1kYXRhLXNjaWVuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNzU2OWI2ZjIyMmRhNjA5NjA0NDU5OTIxZjAxNTk4MWMwNWI4YjQ1OWUyZWYzNjk5YjY4MWFkYjY4ZjE3YWVhNCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzMzExMTdkMzRjNDllNmNhY2JkNmE5ZmU0MjhiZjQzM2RlODNlY2I1MmRkMzJlNTEwYmU4Y2VhZjljZTYwZDI1IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlNGJiOWMzODljNmM3MGVjMGQ5MDkxNmExOGU5NDJlZTM5Y2I5ZmE3N2ExNjQ0NzlmMDcwOTk0ZTdkZDBlYzI3IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjJkNzVhYmEwZmE0ZmUxOWI4ODU2NTMyODBlMWE1MmQxM2IwN2EwZTEzM2RjNDM0ZDFiMmJkN2FiMGY5ZjZiYjgiLAogICAgICAgICJuYW1lIjogImFjY2VwdC9iYXRjaF9pbmZlcmVuY2UubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmZWE3YmJkODY1YjgzNjY0MTEwN2Y3NDAzNDgyNmIxZjU0YTE0ZDE0ZjM4YjE3ZWFjMGZkZGE4MDg5NDEyMmQxIiwKICAgICAgICAibmFtZSI6ICJhY2NlcHQvaW50YWtlX2NvbnRyYWN0Lm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDNiNGM5MDc0MTVlYTQzNjBlMDlhNGFhMjJlOWNhOGU2OGJkMjNmOGI0YzQ3Y2NmOWQxYjk1ZDhhYTYxYmU5ZiIsCiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L3ByZWZsaWdodC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImY5ZjJkZjljYWE0ZDU2OWU0M2U0ZDM0ZjcwOTliMjE1MTFmYWMxYWIzOWVkNjIyNzMyNDhmYjYxMWI5Y2JkM2UiLAogICAgICAgICJuYW1lIjogImFjY2VwdC9zZW1hbnRpY192aWV3Lm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjY0MDRkOWNiMTNkMDA5M2I2NzAxZGZkNGIxOTMzNzEwM2Q4ZDNiODYwNzVlMTExMzkzNmQzYjQ0ZTk5Y2NhOSIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImM2YTM2ODQ1NGZhOWFkZWViYTBiZGRhOThiZDNkMjE0ZjJiOTJhNWYxMjgzNTI3NDVjM2QyMmViOTA3YjViNmIiLAogICAgICAgICJuYW1lIjogImhvc3RzL2dlbmVyaWMvYWRhcHRlci5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjBjZDA0NzkzNDNiNmM4MWE1YWU5ZjI4Zjk5N2ZlMGZhMDYxZGFjODA1ZDI0ODlmZGUxMTg2NzMxZjBlMDE2ZGYiLAogICAgICAgICJuYW1lIjogImhvc3RzL2dlbmVyaWMvcnVudGltZS5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImM1ZWRlYTBlYzhiNzk1MmI4ZmU1ZTViYTZjMDcwMjUwNzgwMjI5MTBjN2U1MDU4ZmMwMDAwYWE2NDY3MGY1M2UiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2NTA2ZTY2MWJjMTFjMjVlYjY2YTdiN2MzNWEzNDAzMTM5NTdkMzI0YmEzODY3ZTk1MDg2YjExNGU5MmQyMDRhIiwKICAgICAgICAibmFtZSI6ICJ2YWxpZGF0ZS9hdG9taWNfcXVlc3Rpb25fY2hlY2subWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3YmQzOTM0ZTFiOWMyMmE0YzM4YjM3YmNiOGM1OTcyOTJkZGVlODIyMDYxZDlmZWVlZTY1N2JkOTA1MDM1Y2Q0IiwKICAgICAgICAibmFtZSI6ICJ2YWxpZGF0ZS9leGFtcGxlcy9iYWRfcmVxdWVzdC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjg5MmQwZmJlYjQyYTgwMmQzMzZmZmIyMzA5ZDVlOTk5YjM4Y2ZiNDg3ZDM2OWMzODkyOTQ5ZjNjYzFiZWQ4ZTYiLAogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2V4YW1wbGVzL2dvb2RfcmVxdWVzdC5tZCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMCnuu90cyjzIwuc9siUNkR3LgxyEXxXp7ZBbsYxVpxOTQJbQY5DLH+3SZuhCFQo3UAIxAK05uplVBCVMowcOKH7kA6x6kDvgAsK/ju6HXHrccHIqkTfln11i2wRU6mcRQnPu0w==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-graph-visualization/BENCHMARK.md b/skills/kumo-graph-visualization/BENCHMARK.md index a5513ca7a..c78575965 100644 --- a/skills/kumo-graph-visualization/BENCHMARK.md +++ b/skills/kumo-graph-visualization/BENCHMARK.md @@ -11,7 +11,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-graph-visualization` - Validation status: `passed` - Evaluation date: 2026-11-18 -- Evaluator version: `1.6.9` +- Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 5 evaluation tasks (4 positive, 1 negative) - Dataset digest: `sha256:5ce8bff0dfb494254338b949f512aa368984b20ef512d553ecdf7175e11ea4c8` (skill-evaluator-dataset-snapshot/1) @@ -28,11 +28,11 @@ SkillSpector provider, model, analyzer revision, and determinism evidence retain | Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | | --- | --- | --- | --- | --- | --- | --- | -| kumo-graph-visualization | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-graph-visualization | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-graph-visualization | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-graph-visualization | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-graph-visualization | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-graph-visualization | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-graph-visualization | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-graph-visualization | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-graph-visualization | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-graph-visualization | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | - Controls for `kumo-graph-visualization`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.1% — baseline ran, but no comparable score was available; uplift unavailable | 79.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 90.7% — baseline ran, but no comparable score was available; uplift unavailable | 79.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 68.0% (+40.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 30.8% → 50.0% (+19.2 points) | 30.0% → 37.5% (+7.5 points) | -| Efficiency | 91.5% — baseline ran, but no comparable score was available; uplift unavailable | 96.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 32.0% → 100.0% (+68.0 points) | 8.0% → 72.0% (+64.0 points) | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 91.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 30.8% → 60.0% (+29.2 points) | 13.3% → 37.5% (+24.2 points) | +| Efficiency | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | 96.4% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 89.9% → 97.7% (+7.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 92.6% → 97.5% (+4.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.3% → 93.0% (+1.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 93.3% → 92.9% (-0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 804,052 | 715,405 | +88,647 | +12.39% | skill 5/5; base 5/5 | -| claude-code | kviz-01 | 145,945 | 143,072 | +2,873 | +2.01% | skill 1/1; base 1/1 | -| claude-code | kviz-02 | 147,079 | 107,442 | +39,637 | +36.89% | skill 1/1; base 1/1 | -| claude-code | kviz-03 | 185,748 | 179,161 | +6,587 | +3.68% | skill 1/1; base 1/1 | -| claude-code | kviz-04 | 110,109 | 107,697 | +2,412 | +2.24% | skill 1/1; base 1/1 | -| claude-code | kviz-05 | 215,171 | 178,033 | +37,138 | +20.86% | skill 1/1; base 1/1 | -| codex | All cases | 241,390 | 257,645 | -16,255 | -6.31% | skill 5/5; base 5/5 | -| codex | kviz-01 | 57,543 | 68,459 | -10,916 | -15.95% | skill 1/1; base 1/1 | -| codex | kviz-02 | 27,456 | 46,484 | -19,028 | -40.93% | skill 1/1; base 1/1 | -| codex | kviz-03 | 57,025 | 40,586 | +16,439 | +40.50% | skill 1/1; base 1/1 | -| codex | kviz-04 | 42,204 | 61,244 | -19,040 | -31.09% | skill 1/1; base 1/1 | -| codex | kviz-05 | 57,162 | 40,872 | +16,290 | +39.86% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,045,442 | 973,050 | +72,392 | +7.44% | skill 10/10; base 10/10 | +| claude-code | All cases | 808,922 | 788,473 | +20,449 | +2.59% | skill 5/5; base 5/5 | +| claude-code | kviz-01 | 145,987 | 177,996 | -32,009 | -17.98% | skill 1/1; base 1/1 | +| claude-code | kviz-02 | 147,637 | 107,276 | +40,361 | +37.62% | skill 1/1; base 1/1 | +| claude-code | kviz-03 | 147,145 | 182,178 | -35,033 | -19.23% | skill 1/1; base 1/1 | +| claude-code | kviz-04 | 109,826 | 107,310 | +2,516 | +2.34% | skill 1/1; base 1/1 | +| claude-code | kviz-05 | 258,327 | 213,713 | +44,614 | +20.88% | skill 1/1; base 1/1 | +| codex | All cases | 272,278 | 205,989 | +66,289 | +32.18% | skill 5/5; base 5/5 | +| codex | kviz-01 | 57,309 | 40,444 | +16,865 | +41.70% | skill 1/1; base 1/1 | +| codex | kviz-02 | 42,456 | 57,036 | -14,580 | -25.56% | skill 1/1; base 1/1 | +| codex | kviz-03 | 73,002 | 27,052 | +45,950 | +169.86% | skill 1/1; base 1/1 | +| codex | kviz-04 | 57,045 | 41,055 | +15,990 | +38.95% | skill 1/1; base 1/1 | +| codex | kviz-05 | 42,466 | 40,402 | +2,064 | +5.11% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,081,200 | 994,462 | +86,738 | +8.72% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-graph-visualization/skill-card.md b/skills/kumo-graph-visualization/skill-card.md index f358e0ebc..5c5d231c7 100644 --- a/skills/kumo-graph-visualization/skill-card.md +++ b/skills/kumo-graph-visualization/skill-card.md @@ -42,7 +42,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`), each compared against a no-skill baseline. Evaluator version `1.6.9`, evaluation date 2026-11-18.
+5 evaluation tasks (4 positive, 1 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`), each compared against a no-skill baseline. Evaluator version `1.6.10`, evaluation date 2026-11-18.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -68,12 +68,12 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.1% — baseline ran, but no comparable score was available; uplift unavailable | 79.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 90.7% — baseline ran, but no comparable score was available; uplift unavailable | 79.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 68.0% (+40.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 30.8% → 50.0% (+19.2 points) | 30.0% → 37.5% (+7.5 points) | -| Efficiency | 91.5% — baseline ran, but no comparable score was available; uplift unavailable | 96.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 32.0% → 100.0% (+68.0 points) | 8.0% → 72.0% (+64.0 points) | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 91.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 30.8% → 60.0% (+29.2 points) | 13.3% → 37.5% (+24.2 points) | +| Efficiency | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | 96.4% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -88,9 +88,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 89.9% → 97.7% (+7.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 92.6% → 97.5% (+4.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.3% → 93.0% (+1.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 93.3% → 92.9% (-0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -104,7 +104,7 @@ Shared-source comparisons use the same signals and weights on both conditions. T **Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report.
## Skill Version(s):
-0.1.0 (source: pyproject.toml; git SHA c7554e32, committed 2026-11-17)
+0.1.0 (source: pyproject.toml; git SHA 7cbda7af, committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-graph-visualization/skill.oms.sig b/skills/kumo-graph-visualization/skill.oms.sig index 83bbbc5b4..cbd140d0e 100644 --- a/skills/kumo-graph-visualization/skill.oms.sig +++ b/skills/kumo-graph-visualization/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1ncmFwaC12aXN1YWxpemF0aW9uIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImJlZmMwZTkyOTE5MDAwM2U1YWRmZWU4ODUyYzMzY2Q3ZmJmOTQ3NzMyOWVmNDE1YWMxMTA4ZWQ3OTliMzRlMDUiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0KICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMDg2NjhmNTg3NjdlY2Q5Njk2NDc5ODg4YTZkMTAyYzA1OWY1NzY4ZjhjOGNiZGE4ZGFlNDI5YzY3YzMwOWJhYSIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOTU2M2MyZDVkNGQzMmJhYjAyOGE0Zjk2YmExNWU0NDVkN2FlYzIyZjEzZjRkYmY0NDc2NWVjZjUzYjBjOGZmNyIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3NzRhMjc4YTY2OTBmZTBiZjcxMDAxZDc2MDk1ZDM2NmQxMGE4MDNjOGE2Njg2NWRkZjM0YTMyZTc1ZTZiOTliIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNGEzMThkYTdmZmMxOTI3MTgzOTQ4NmJkZDE1NDNkMjhhY2NmMmM2MGEyZGU0NDVmMTdkNGQzYWU0YTJlMGVmOSIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMEX2cvnKYaf3YEDmv4qUDYx2WGb13J1tEJ+zljdQP9azsUzwvPKTmxIBmJpqXD1ecAIwUqZeNXxq8M1N360XeASg5Xdmq6LoM3W3LH1X/oEoBFgF76HoFD07ZGVSrDVR2e42","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1ncmFwaC12aXN1YWxpemF0aW9uIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImJkYjViYTFmYWUxMzA0YTY0OTFmYjlhYzEwYjRiNTkyNzliMjVlYThmYTY2ZjlhYTM5NDU0ZTA2NDE2ODUxNTEiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImNjYjYwNDM3YTFlMGMzNWM2ZDc4NjNlMDlkOTBmNGNmN2U1OWY4ODUxZGM5ODFjYzEzYWRjYTM0ZmQ3M2IzZmEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOTU2M2MyZDVkNGQzMmJhYjAyOGE0Zjk2YmExNWU0NDVkN2FlYzIyZjEzZjRkYmY0NDc2NWVjZjUzYjBjOGZmNyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjc3NGEyNzhhNjY5MGZlMGJmNzEwMDFkNzYwOTVkMzY2ZDEwYTgwM2M4YTY2ODY1ZGRmMzRhMzJlNzVlNmI5OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkMzFiOTk3MGUxNmY1ZGRjMDRjMDU1NmVkMzUzOTgzYjIyMzI3MjhkNmI0ZmQ4MGMyZmQ5ZjAyMzlmZmJiMGY4IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIKICAgICAgXSwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDBw6wiYYjBbzGmdGkCOSuNrRTlN7rH01TPGF7qphMNKNEfoQt7uBN35qhK5Sn0oWcCMQDVywIMWOB51cgKBowea1PIcbRXCeGH5Xq995h6v6k27Y/sxt2HFtOmuj9rUqGoejY=","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-explanations/BENCHMARK.md b/skills/kumo-model-explanations/BENCHMARK.md index ac49beb9d..e53d9c40b 100644 --- a/skills/kumo-model-explanations/BENCHMARK.md +++ b/skills/kumo-model-explanations/BENCHMARK.md @@ -11,7 +11,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-model-explanations` - Validation status: `passed` - Evaluation date: 2026-11-18 -- Evaluator version: `1.6.9` +- Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) - Dataset digest: `sha256:e389a4317a270919c8e56c8d8f40c51460e1ecc5bef453075887cea72bad3559` (skill-evaluator-dataset-snapshot/1) @@ -28,11 +28,11 @@ SkillSpector provider, model, analyzer revision, and determinism evidence retain | Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | | --- | --- | --- | --- | --- | --- | --- | -| kumo-model-explanations | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-explanations | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-explanations | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-explanations | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-explanations | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-explanations | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-explanations | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-explanations | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-explanations | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-explanations | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | - Controls for `kumo-model-explanations`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 91.7% — baseline ran, but no comparable score was available; uplift unavailable | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.5% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 46.7% → 100.0% (+53.3 points) | 63.3% → 93.3% (+30.0 points) | +| Correctness | 43.3% → 100.0% (+56.7 points) | 56.7% → 93.3% (+36.6 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 29.2% → 60.4% (+31.2 points) | 41.7% → 57.0% (+15.3 points) | -| Efficiency | 97.9% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 25.0% → 60.4% (+35.4 points) | 32.0% → 68.1% (+36.1 points) | +| Efficiency | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.2% → 95.8% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 95.9% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 88.9% → 92.0% (+3.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.3% → 91.9% (+2.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,21 +80,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 927,473 | 787,744 | +139,729 | +17.74% | skill 6/6; base 6/6 | -| claude-code | kexpl-01 | 158,137 | 142,432 | +15,705 | +11.03% | skill 1/1; base 1/1 | -| claude-code | kexpl-02 | 159,124 | 108,697 | +50,427 | +46.39% | skill 1/1; base 1/1 | -| claude-code | kexpl-03 | 115,976 | 143,259 | -27,283 | -19.04% | skill 1/1; base 1/1 | -| claude-code | kexpl-04 | 159,980 | 107,413 | +52,567 | +48.94% | skill 1/1; base 1/1 | -| claude-code | kexpl-05 | 226,109 | 178,383 | +47,726 | +26.75% | skill 1/1; base 1/1 | -| claude-code | kexpl-06 | 108,147 | 107,560 | +587 | +0.55% | skill 1/1; base 1/1 | -| codex | All cases | 410,225 | 297,492 | +112,733 | +37.89% | skill 6/6; base 6/6 | -| codex | kexpl-01 | 81,280 | 43,931 | +37,349 | +85.02% | skill 1/1; base 1/1 | -| codex | kexpl-02 | 83,129 | 53,527 | +29,602 | +55.30% | skill 1/1; base 1/1 | -| codex | kexpl-03 | 80,951 | 60,563 | +20,388 | +33.66% | skill 1/1; base 1/1 | -| codex | kexpl-04 | 81,288 | 57,670 | +23,618 | +40.95% | skill 1/1; base 1/1 | -| codex | kexpl-05 | 42,650 | 41,123 | +1,527 | +3.71% | skill 1/1; base 1/1 | -| codex | kexpl-06 | 40,927 | 40,678 | +249 | +0.61% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,337,698 | 1,085,236 | +252,462 | +23.26% | skill 12/12; base 12/12 | +| claude-code | All cases | 891,246 | 716,277 | +174,969 | +24.43% | skill 6/6; base 6/6 | +| claude-code | kexpl-01 | 158,307 | 70,650 | +87,657 | +124.07% | skill 1/1; base 1/1 | +| claude-code | kexpl-02 | 159,005 | 108,056 | +50,949 | +47.15% | skill 1/1; base 1/1 | +| claude-code | kexpl-03 | 158,205 | 107,250 | +50,955 | +47.51% | skill 1/1; base 1/1 | +| claude-code | kexpl-04 | 160,396 | 144,461 | +15,935 | +11.03% | skill 1/1; base 1/1 | +| claude-code | kexpl-05 | 148,103 | 178,444 | -30,341 | -17.00% | skill 1/1; base 1/1 | +| claude-code | kexpl-06 | 107,230 | 107,416 | -186 | -0.17% | skill 1/1; base 1/1 | +| codex | All cases | 412,061 | 335,024 | +77,037 | +22.99% | skill 6/6; base 6/6 | +| codex | kexpl-01 | 80,627 | 70,726 | +9,901 | +14.00% | skill 1/1; base 1/1 | +| codex | kexpl-02 | 82,514 | 41,509 | +41,005 | +98.79% | skill 1/1; base 1/1 | +| codex | kexpl-03 | 80,965 | 60,631 | +20,334 | +33.54% | skill 1/1; base 1/1 | +| codex | kexpl-04 | 81,379 | 64,915 | +16,464 | +25.36% | skill 1/1; base 1/1 | +| codex | kexpl-05 | 45,308 | 56,654 | -11,346 | -20.03% | skill 1/1; base 1/1 | +| codex | kexpl-06 | 41,268 | 40,589 | +679 | +1.67% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,303,307 | 1,051,301 | +252,006 | +23.97% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-explanations/skill-card.md b/skills/kumo-model-explanations/skill-card.md index f9bf69206..b8516d5ca 100644 --- a/skills/kumo-model-explanations/skill-card.md +++ b/skills/kumo-model-explanations/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Inspect and interpret gradient attribution, column analysis and sampled explanation subgraphs from a Kumo model trained in a Python notebook.
+Inspect and interpret gradient attribution, column analysis, and sampled explanation subgraphs from a Kumo model trained in a Python notebook, locating the correct run's explanation artifacts and identifying missing or unsupported explanations without automatically retraining.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill with coding agents to read the explanation artifacts of an existing Kumo run, such as which table columns the model relied on and how predictions compare with outcomes across column cohorts, without retraining.
+Developers and data scientists use this skill to inspect a Kumo model's retained explanation artifacts in a notebook, explain gradient attribution scores and column-analysis cohorts, and diagnose missing or unsupported explanations without starting another fit.
### Deployment Geography for Use:
Global
@@ -25,15 +25,15 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/artifacts.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-explanations/references/artifacts.md)
-- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+- [Explanation artifact workflow](references/artifacts.md)
+- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
## Skill Output:
-**Output Type(s):** [Code, Analysis]
-**Output Format:** [Markdown assessment with inline Python code blocks and plots]
+**Output Type(s):** [Analysis, Code]
+**Output Format:** [Markdown with inline Python code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Inspection only; makes no causal claims and starts no fit.]
+**Other Properties Related to Output:** [None]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -42,7 +42,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.9`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`), compared against a no-skill baseline. Evaluator version `1.6.10`; evaluation date 2026-11-18.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -64,7 +64,7 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
-> ✅ **Overall verdict: PASS — Recommended for publication** +**Overall verdict:** ✅ PASS — Recommended for publication **Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. @@ -72,38 +72,28 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 91.7% — baseline ran, but no comparable score was available; uplift unavailable | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.5% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 46.7% → 100.0% (+53.3 points) | 63.3% → 93.3% (+30.0 points) | +| Correctness | 43.3% → 100.0% (+56.7 points) | 56.7% → 93.3% (+36.6 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 29.2% → 60.4% (+31.2 points) | 41.7% → 57.0% (+15.3 points) | -| Efficiency | 97.9% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 25.0% → 60.4% (+35.4 points) | 32.0% → 68.1% (+36.1 points) | +| Efficiency | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | -**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. +Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. -A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. - -#### Comparison details +### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.2% → 95.8% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 95.9% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 88.9% → 92.0% (+3.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | - -### Tier Status - -| Tier | Purpose | Status | Evidence | -|---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | -| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | +| codex | Efficiency | Token-only comparison | 89.3% → 91.9% (+2.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
-c7554e32 (source: git SHA, committed 2026-11-17); repository pyproject.toml version 0.1.0
+0.1.0 (source: pyproject.toml; git SHA 7cbda7af, committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-model-explanations/skill.oms.sig b/skills/kumo-model-explanations/skill.oms.sig index e832b2c04..a07d73fd4 100644 --- a/skills/kumo-model-explanations/skill.oms.sig +++ b/skills/kumo-model-explanations/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1leHBsYW5hdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYWQzOGI1NDBhMTU0MTA2MjZlY2NjN2NkNzFmMWI5YjM2NDZhZDFjZTQzNzFhYjA4YWYyZmVkZTY5MmQxYmNmYyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzYzRmZmQyNTJjYWY3Y2E1N2I3ZWQzMDQ2MjUzZTA3YTMzZGEwNDZmMjI1MmY1ZDYwZjlmMDIyOThiNTRhNTEyIiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJjYjc0YjdkZGNkZDNhZTA2YzUyYmM4NTJjZDljOTc5ZTk0YmE2NDBjNmVjZTgyMmY1ZjQ5MmJiZmY5N2RhYTBjIiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjk2NzRhNzFhMWExYmYwNmU1ZjNhMDdlNGNjZWRlNTdkMTMzNjYzOWY0NGMyMWYwMzQzYWQ3MmE3ZTA3MmRiMTciLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI1MGNiNjhmYWJkNzRjOGUxZTkyYzRjODMwZWU1MDllZGNiNjY1NGI3MGY5OGMyN2NlOGZlMzRjODk4ZWE3MTdhIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2FydGlmYWN0cy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjNlOTU4ZGJiNWVlOWQ1YjA1YjRjMTc2NDk0NjE3NWU3YmUxZjI3NTFhY2U3NDFiMmE2MDJlYzY3NTEwZjQyZjMiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMD2y1HVohgd0Hadvyl6V8B2Kmycfk+L30bDQr8g9KBCdWzvzYR+9cBfzU5BvwsGg4gIwb4vOX1RgNRbH/tV4PrMa2l33SChf4K47ASVVXTu9NrcQiWpxgMId+e5yyoJiA8oH","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1leHBsYW5hdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNWIwNjUxZmFjNzliZjUxMWUxZjIyMzQyNmJhZmQwNzYwMzcwNzNhNzFhMGRmZWE3NDRiYzNjZjc2ZWM3ZGUxMyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjUzNjRmM2MzYzZkYTY3ZjUzM2UzMDRkNDUwZDgyODFmNGVkZjFkMTNlNDRiZmJhMGQ2ZDUxZjExYTkzZTc5YmUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiY2I3NGI3ZGRjZGQzYWUwNmM1MmJjODUyY2Q5Yzk3OWU5NGJhNjQwYzZlY2U4MjJmNWY0OTJiYmZmOTdkYWEwYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjk2NzRhNzFhMWExYmYwNmU1ZjNhMDdlNGNjZWRlNTdkMTMzNjYzOWY0NGMyMWYwMzQzYWQ3MmE3ZTA3MmRiMTciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9hcnRpZmFjdHMubWQiLAogICAgICAgICJkaWdlc3QiOiAiNTBjYjY4ZmFiZDc0YzhlMWU5MmM0YzgzMGVlNTA5ZWRjYjY2NTRiNzBmOThjMjdjZThmZTM0Yzg5OGVhNzE3YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjkzM2M5ZDk5YzRkMGU3NDFkNGI3ZDA3ZjVjYzc3OTI3OGMxZTQ5MDIxNDEyMzA0OTlhMjY5MDlmNGNjY2JmOGQiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQC1NAaO59p2CHGCR4qHXfcn/Qc6MSgWSW+TFtkftb0UdoBbk4t4JPpmiZb7zHXecR4CMQDhAtGyvvTpYmaHMa32vrH5Cp3uWtX/49EAWaf8gFFXTrshgyhs6AoIkUy2WVWHPl8=","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-persistence/BENCHMARK.md b/skills/kumo-model-persistence/BENCHMARK.md index 8aac466ed..aa524bbbd 100644 --- a/skills/kumo-model-persistence/BENCHMARK.md +++ b/skills/kumo-model-persistence/BENCHMARK.md @@ -11,7 +11,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-model-persistence` - Validation status: `passed` - Evaluation date: 2026-11-18 -- Evaluator version: `1.6.9` +- Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) - Dataset digest: `sha256:096c403e279883d033c46cf48b03d51e3a6ade8a0c67149a36066124537ddb47` (skill-evaluator-dataset-snapshot/1) @@ -28,11 +28,11 @@ SkillSpector provider, model, analyzer revision, and determinism evidence retain | Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | | --- | --- | --- | --- | --- | --- | --- | -| kumo-model-persistence | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-persistence | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-persistence | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-persistence | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-persistence | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-persistence | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-persistence | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-persistence | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-persistence | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-persistence | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | - Controls for `kumo-model-persistence`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 87.6% — baseline ran, but no comparable score was available; uplift unavailable | 79.5% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 50.0% → 96.7% (+46.7 points) | 36.7% → 76.7% (+40.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 35.0% → 47.2% (+12.2 points) | 13.9% → 61.1% (+47.2 points) | -| Efficiency | 98.4% — baseline ran, but no comparable score was available; uplift unavailable | 91.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 53.3% → 96.7% (+43.4 points) | 36.7% → 73.3% (+36.6 points) | +| Discoverability | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 26.7% → 50.6% (+23.9 points) | 30.6% → 44.5% (+13.9 points) | +| Efficiency | 94.8% — baseline ran, but no comparable score was available; uplift unavailable | 92.6% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.9% → 96.8% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 96.9% (-1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.4% → 92.3% (+0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 91.2% → 91.9% (+0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,21 +80,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,019,743 | 830,788 | +188,955 | +22.74% | skill 6/6; base 6/6 | -| claude-code | kpersist-01 | 189,044 | 144,899 | +44,145 | +30.47% | skill 1/1; base 1/1 | -| claude-code | kpersist-02 | 188,504 | 184,161 | +4,343 | +2.36% | skill 1/1; base 1/1 | -| claude-code | kpersist-03 | 194,119 | 143,156 | +50,963 | +35.60% | skill 1/1; base 1/1 | -| claude-code | kpersist-04 | 269,404 | 144,174 | +125,230 | +86.86% | skill 1/1; base 1/1 | -| claude-code | kpersist-05 | 71,208 | 106,978 | -35,770 | -33.44% | skill 1/1; base 1/1 | -| claude-code | kpersist-06 | 107,464 | 107,420 | +44 | +0.04% | skill 1/1; base 1/1 | -| codex | All cases | 445,785 | 303,864 | +141,921 | +46.71% | skill 6/6; base 6/6 | -| codex | kpersist-01 | 93,061 | 41,296 | +51,765 | +125.35% | skill 1/1; base 1/1 | -| codex | kpersist-02 | 42,478 | 41,293 | +1,185 | +2.87% | skill 1/1; base 1/1 | -| codex | kpersist-03 | 131,524 | 55,298 | +76,226 | +137.85% | skill 1/1; base 1/1 | -| codex | kpersist-04 | 95,555 | 64,736 | +30,819 | +47.61% | skill 1/1; base 1/1 | -| codex | kpersist-05 | 42,036 | 60,287 | -18,251 | -30.27% | skill 1/1; base 1/1 | -| codex | kpersist-06 | 41,131 | 40,954 | +177 | +0.43% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,465,528 | 1,134,652 | +330,876 | +29.16% | skill 12/12; base 12/12 | +| claude-code | All cases | 1,095,881 | 752,122 | +343,759 | +45.71% | skill 6/6; base 6/6 | +| claude-code | kpersist-01 | 269,744 | 143,720 | +126,024 | +87.69% | skill 1/1; base 1/1 | +| claude-code | kpersist-02 | 228,074 | 107,402 | +120,672 | +112.36% | skill 1/1; base 1/1 | +| claude-code | kpersist-03 | 230,584 | 143,145 | +87,439 | +61.08% | skill 1/1; base 1/1 | +| claude-code | kpersist-04 | 188,567 | 143,493 | +45,074 | +31.41% | skill 1/1; base 1/1 | +| claude-code | kpersist-05 | 71,305 | 106,857 | -35,552 | -33.27% | skill 1/1; base 1/1 | +| claude-code | kpersist-06 | 107,607 | 107,505 | +102 | +0.09% | skill 1/1; base 1/1 | +| codex | All cases | 398,521 | 311,624 | +86,897 | +27.89% | skill 6/6; base 6/6 | +| codex | kpersist-01 | 77,307 | 72,416 | +4,891 | +6.75% | skill 1/1; base 1/1 | +| codex | kpersist-02 | 42,201 | 13,428 | +28,773 | +214.28% | skill 1/1; base 1/1 | +| codex | kpersist-03 | 98,036 | 61,157 | +36,879 | +60.30% | skill 1/1; base 1/1 | +| codex | kpersist-04 | 92,901 | 62,029 | +30,872 | +49.77% | skill 1/1; base 1/1 | +| codex | kpersist-05 | 46,728 | 61,509 | -14,781 | -24.03% | skill 1/1; base 1/1 | +| codex | kpersist-06 | 41,348 | 41,085 | +263 | +0.64% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,494,402 | 1,063,746 | +430,656 | +40.48% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-persistence/skill-card.md b/skills/kumo-model-persistence/skill-card.md index ee47bf529..2fe87ed7f 100644 --- a/skills/kumo-model-persistence/skill-card.md +++ b/skills/kumo-model-persistence/skill-card.md @@ -25,9 +25,9 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/reuse.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/references/reuse.md)
-- [references/retention.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/references/retention.md)
-- [scripts/retain_artifacts.py](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-persistence/scripts/retain_artifacts.py)
+- [references/reuse.md](references/reuse.md)
+- [references/retention.md](references/retention.md)
+- [scripts/retain_artifacts.py](scripts/retain_artifacts.py)
- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
@@ -45,7 +45,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.9`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.10`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -77,17 +77,15 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | 84.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 87.6% — baseline ran, but no comparable score was available; uplift unavailable | 79.5% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 50.0% → 96.7% (+46.7 points) | 36.7% → 76.7% (+40.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 35.0% → 47.2% (+12.2 points) | 13.9% → 61.1% (+47.2 points) | -| Efficiency | 98.4% — baseline ran, but no comparable score was available; uplift unavailable | 91.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 53.3% → 96.7% (+43.4 points) | 36.7% → 73.3% (+36.6 points) | +| Discoverability | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 26.7% → 50.6% (+23.9 points) | 30.6% → 44.5% (+13.9 points) | +| Efficiency | 94.8% — baseline ran, but no comparable score was available; uplift unavailable | 92.6% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. -Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. - A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. #### Comparison details @@ -97,9 +95,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.9% → 96.8% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 96.9% (-1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.4% → 92.3% (+0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 91.2% → 91.9% (+0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -110,7 +108,7 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | ## Skill Version(s):
-c7554e32 (source: git SHA, committed 2026-11-17); repository pyproject.toml version 0.1.0
+0.1.0 (source: pyproject.toml; git SHA 7cbda7af, committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-model-persistence/skill.oms.sig b/skills/kumo-model-persistence/skill.oms.sig index 34f014cac..43ba1bfcf 100644 --- a/skills/kumo-model-persistence/skill.oms.sig +++ b/skills/kumo-model-persistence/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1wZXJzaXN0ZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIxNWZhOWE5Y2Q0ZDkxZDA4MDc4OTI0NGY3NjgxZTYxNzc0MzBjMjIxNGM2Yzc3ZTUyOGE1MzEwZmRjNGU4MzBhIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiMjY3ZmMzNDFhNjZmZmU3NjUxNmRmNDY4MDYwMzhkMmY1ZDY2YWIyMzJiZTU1ZDI0Nzc0ZGZlMWM4YTVhZTdlYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJiMWU4MWU2ODg2ZDNkYTczZmMwMzI2ZjhmMGQ1YTYzYTBjNzIzNjlkNmYyNWZkM2M0MjkyN2U5YWUyMDBlZTI4IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiMzY3ZTg3MWE3MmUzZTZjN2I1MWRhYjlhMzJiMzMwOTQyZDAwMjllZTY0NzNjMzM3NjcyMzI2NGQyMmYzY2E2YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3JldGVudGlvbi5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIyYjA1NGFmMzNmODE2MmM1MThkZWE5ZjdhZDg0NGMwZDQxNjA3YjAwM2Q1ODIxZDUwYTJiZjY4YmU2Y2FjZjMyIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcmV1c2UubWQiLAogICAgICAgICJkaWdlc3QiOiAiMmM1YTUwMzFjODUwMTlmMTIxNWU2Yzc0MGRjZWRlN2E2ZTkyMDg1MmIzZjU3MmYwNGY5NGRhMGVjY2M4YWU4MSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JldGFpbl9hcnRpZmFjdHMucHkiLAogICAgICAgICJkaWdlc3QiOiAiNGQzZDAzYjEwODlhMDI0MGQxZjkxM2MxZWQyZjhlNGFmOTg5ZWNiOGQzZTE5OWMzMmZkZGFlOTBmY2JhYWQzYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjg4OGExNDcwM2QyNTQ3OWJjZjYwYjhjMzI0ZWI0MTI3ZGIyYjgwMTIwM2JjMmNiYjhjMGYwYzhmZTBhZTRmNjgiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMGhSpnlbYzj/ttppt3sbMtkii3c2wBbfu8/Z0P1RGuf5UefvzGs4e2sFpJttI8QTFwIwe8FOIzE0F9lUuIJS90dU79X/otq5xuHnj7f+bEgT/pIzKv2X8k6bK0WqI2LqN9ds","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1wZXJzaXN0ZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIxNTExYWEwNGM5NWNjYjYwOGMzNGFmMmYxODI0MmE3ZGZmN2EwNGJhOTRkZjZkZDQ4YzE3OWQxMDM5NDM0MmU3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjMzZmMzZDRjM2VkOThiNGQ2NGFhYWQ1YzExYmJlNWYwOGUzY2ViNGU0ZmYzZjRmMWVhYzcxNzliZGJhYjI4YmUiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiYjFlODFlNjg4NmQzZGE3M2ZjMDMyNmY4ZjBkNWE2M2EwYzcyMzY5ZDZmMjVmZDNjNDI5MjdlOWFlMjAwZWUyOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjM2N2U4NzFhNzJlM2U2YzdiNTFkYWI5YTMyYjMzMDk0MmQwMDI5ZWU2NDczYzMzNzY3MjMyNjRkMjJmM2NhNmEiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9yZXRlbnRpb24ubWQiLAogICAgICAgICJkaWdlc3QiOiAiMmIwNTRhZjMzZjgxNjJjNTE4ZGVhOWY3YWQ4NDRjMGQ0MTYwN2IwMDNkNTgyMWQ1MGEyYmY2OGJlNmNhY2YzMiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3JldXNlLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjJjNWE1MDMxYzg1MDE5ZjEyMTVlNmM3NDBkY2VkZTdhNmU5MjA4NTJiM2Y1NzJmMDRmOTRkYTBlY2NjOGFlODEiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXRhaW5fYXJ0aWZhY3RzLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjRkM2QwM2IxMDg5YTAyNDBkMWY5MTNjMWVkMmY4ZTRhZjk4OWVjYjhkM2UxOTljMzJmZGRhZTkwZmNiYWFkM2IiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI5MzRlMTRiZWJkMmFiNTMyMmM3YmFmYmZhNmJkNzFlZWQ2MmZmYjNlOTAyMDZiMGQxZDYwOTI5MGM4NjU5OWY5IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMHTQCjGe5J4FqdfHSg9Kctz2/ss2/F0xkiHXwdNzezDoHD48Wswkt30med75qhWgawIwNddwEMf5RoLTNgn388Ri89neGNK58++zqSxTebFj8sya/BOKuBIPpISR1izIEHlL","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-tuning/BENCHMARK.md b/skills/kumo-model-tuning/BENCHMARK.md index 57f3c6fc7..5bfb28313 100644 --- a/skills/kumo-model-tuning/BENCHMARK.md +++ b/skills/kumo-model-tuning/BENCHMARK.md @@ -11,7 +11,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-model-tuning` - Validation status: `passed` - Evaluation date: 2026-11-18 -- Evaluator version: `1.6.9` +- Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 5 evaluation tasks (4 positive, 1 negative) - Dataset digest: `sha256:7fd78018aceb270831a95ad2a267d9e84f1e0175652850059d7cda7224f540ae` (skill-evaluator-dataset-snapshot/1) @@ -28,11 +28,11 @@ SkillSpector provider, model, analyzer revision, and determinism evidence retain | Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | | --- | --- | --- | --- | --- | --- | --- | -| kumo-model-tuning | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-tuning | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-tuning | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-tuning | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-model-tuning | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-model-tuning | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-tuning | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-tuning | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-tuning | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-model-tuning | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | - Controls for `kumo-model-tuning`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 96.7% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 98.9% — baseline ran, but no comparable score was available; uplift unavailable | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 68.0% → 100.0% (+32.0 points) | 60.0% → 96.0% (+36.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 47.5% → 90.0% (+42.5 points) | 51.7% → 80.8% (+29.1 points) | -| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 76.0% → 100.0% (+24.0 points) | 60.0% → 96.0% (+36.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 62.5% → 100.0% (+37.5 points) | 51.7% → 70.8% (+19.1 points) | +| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 87.6% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.0% → 97.3% (-0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.9% → 97.4% (-0.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.6% → 92.3% (+0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 91.6% → 91.9% (+0.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 808,588 | 831,826 | -23,238 | -2.79% | skill 5/5; base 5/5 | -| claude-code | ktune-01 | 109,085 | 71,246 | +37,839 | +53.11% | skill 1/1; base 1/1 | -| claude-code | ktune-02 | 147,053 | 142,382 | +4,671 | +3.28% | skill 1/1; base 1/1 | -| claude-code | ktune-03 | 147,632 | 178,628 | -30,996 | -17.35% | skill 1/1; base 1/1 | -| claude-code | ktune-04 | 109,109 | 107,304 | +1,805 | +1.68% | skill 1/1; base 1/1 | -| claude-code | ktune-05 | 295,709 | 332,266 | -36,557 | -11.00% | skill 1/1; base 1/1 | -| codex | All cases | 247,372 | 224,339 | +23,033 | +10.27% | skill 5/5; base 5/5 | -| codex | ktune-01 | 41,737 | 48,910 | -7,173 | -14.67% | skill 1/1; base 1/1 | -| codex | ktune-02 | 57,400 | 54,948 | +2,452 | +4.46% | skill 1/1; base 1/1 | -| codex | ktune-03 | 56,884 | 40,383 | +16,501 | +40.86% | skill 1/1; base 1/1 | -| codex | ktune-04 | 45,926 | 35,249 | +10,677 | +30.29% | skill 1/1; base 1/1 | -| codex | ktune-05 | 45,425 | 44,849 | +576 | +1.28% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,055,960 | 1,056,165 | -205 | -0.02% | skill 10/10; base 10/10 | +| claude-code | All cases | 734,363 | 830,668 | -96,305 | -11.59% | skill 5/5; base 5/5 | +| claude-code | ktune-01 | 109,059 | 71,625 | +37,434 | +52.26% | skill 1/1; base 1/1 | +| claude-code | ktune-02 | 146,360 | 218,578 | -72,218 | -33.04% | skill 1/1; base 1/1 | +| claude-code | ktune-03 | 148,102 | 141,918 | +6,184 | +4.36% | skill 1/1; base 1/1 | +| claude-code | ktune-04 | 109,094 | 143,000 | -33,906 | -23.71% | skill 1/1; base 1/1 | +| claude-code | ktune-05 | 221,748 | 255,547 | -33,799 | -13.23% | skill 1/1; base 1/1 | +| codex | All cases | 253,211 | 239,042 | +14,169 | +5.93% | skill 5/5; base 5/5 | +| codex | ktune-01 | 46,169 | 49,399 | -3,230 | -6.54% | skill 1/1; base 1/1 | +| codex | ktune-02 | 72,379 | 54,978 | +17,401 | +31.65% | skill 1/1; base 1/1 | +| codex | ktune-03 | 56,882 | 40,860 | +16,022 | +39.21% | skill 1/1; base 1/1 | +| codex | ktune-04 | 45,835 | 48,842 | -3,007 | -6.16% | skill 1/1; base 1/1 | +| codex | ktune-05 | 31,946 | 44,963 | -13,017 | -28.95% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 987,574 | 1,069,710 | -82,136 | -7.68% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-tuning/skill-card.md b/skills/kumo-model-tuning/skill-card.md index 76c94201d..adee29b54 100644 --- a/skills/kumo-model-tuning/skill-card.md +++ b/skills/kumo-model-tuning/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Configure a Kumo model plan for a user-defined task, including encoders, run mode, bounded experiments, sampling, optimization, architecture, metrics and refits.
+Configure a Kumo model plan for a user-defined task, including encoders, run mode, bounded experiments, sampling, optimization, architecture, metrics and refits, while preserving target and evaluation constraints and using existing diagnostics for comparisons.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill with coding agents to change a Kumo model plan for a confirmed task, such as a smoke test, encoder overrides or bounded experiments, while keeping the target and evaluation protocol unchanged.
+Developers and data scientists use this skill to configure and adjust Kumo Engine model plans (encoders, run mode, experiments, sampling, optimization, architecture, metrics and refits) for controlled experiments without changing task targets or evaluation protocol.
### Deployment Geography for Use:
Global
@@ -25,17 +25,16 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/plans.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-model-tuning/references/plans.md)
-- [Kumo Training Diagnostics skill](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumo-training-diagnostics/SKILL.md)
+- [Public model plans (references/plans.md)](references/plans.md)
+- [Kumo training diagnostics skill](../kumo-training-diagnostics/SKILL.md)
- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
-- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
## Skill Output:
-**Output Type(s):** [Code, Analysis]
+**Output Type(s):** [Code, Configuration instructions, Analysis]
**Output Format:** [Markdown with inline Python code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Suggesting a plan does not run its experiments.]
+**Other Properties Related to Output:** [None]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -44,50 +43,44 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.9`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each attempt in its own isolated sandbox pod (`k8s-sandbox`); each task was also attempted without the skill as a baseline. Evaluator version `1.6.10`, evaluated 2026-11-18.
## Evaluation Metrics Used:
Reported benchmark dimensions:
- Security: Is it safe to use? Scored from `security` (100%).
- Correctness: Is the answer correct? Scored from `accuracy` (100%).
- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
-- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
-- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` (50%) and `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? `skill_efficiency` (50%) + `token_efficiency` (50%); N/A sources are omitted, remaining weights renormalized, and the dimension marked partial.
Underlying evaluation signals used in this run:
-- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Accuracy: final-answer correctness against the reference answer.
-- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
-- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
-- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
> ✅ **Overall verdict: PASS — Recommended for publication** -### Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. ### Results at a Glance | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 96.7% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 98.9% — baseline ran, but no comparable score was available; uplift unavailable | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 68.0% → 100.0% (+32.0 points) | 60.0% → 96.0% (+36.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 47.5% → 90.0% (+42.5 points) | 51.7% → 80.8% (+29.1 points) | -| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | - -**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. - -Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. +| Correctness | 76.0% → 100.0% (+24.0 points) | 60.0% → 96.0% (+36.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 62.5% → 100.0% (+37.5 points) | 51.7% → 70.8% (+19.1 points) | +| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 87.6% — baseline ran, but no comparable score was available; uplift unavailable | -A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. #### Comparison details @@ -96,20 +89,12 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.0% → 97.3% (-0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.9% → 97.4% (-0.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.6% → 92.3% (+0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | - -### Tier Status - -| Tier | Purpose | Status | Evidence | -|---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 11 finding(s) | -| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | +| codex | Efficiency | Token-only comparison | 91.6% → 91.9% (+0.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
-c7554e32 (source: git SHA, committed 2026-11-17); repository pyproject.toml version 0.1.0
+0.1.0 (source: pyproject.toml)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-model-tuning/skill.oms.sig b/skills/kumo-model-tuning/skill.oms.sig index c75a8afbe..e283985cc 100644 --- a/skills/kumo-model-tuning/skill.oms.sig +++ b/skills/kumo-model-tuning/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC10dW5pbmciLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZGY3MjVmY2RkOGQ4ZGY1Mzc1ODBhYjdkOTcyMDQ1MWI4MGM0Yjg2Yjk3OWJmZTNkZWI4YWFlZDViMmIyNjcxZSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2JiYjNjM2YyNzYzNmVkNmNjOTZiOWIwMjE1OGZkNWYwY2U0MGIyNjI1OWYxMDYyN2UyMWYwMjdmY2VhZmM2ZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIxNjQwMWRiY2Y5OTM0ZTY1MTkyNDdhNDNiN2VkYTYxNzc5NjRiMTJmNzBiNjg2MTY0MWU0NDE2MGU2ZDhlN2FiIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiNThiNmNmMzIyNDI1M2VlZTVmOWMxZTAzYjFmODJlM2E5NWVlOGYxYzUyN2MyNWNiYjM1YzE3NTQ4OTBkZGZlYiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BsYW5zLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjNlYjQyYjBmODk1M2QxNzAwMjU1OGQwNWE4ODI4NTE5MzFkOGNiNTEwY2E4ZjdlNWM2MGViOWNmNjc5MDJjNGQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI4Zjk2NmQ3NmJhYTVhZGEyYWFiMjBlODdlZWZkMGYwNDVkZjdjOTQ3OWEyNTIyN2IyNzQ2YzRjMDMwZmU5NmVmIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCJQwv3fRQq8xPrzrwldJ0ix6jeqgvO96yPyz+goo9W6k8aKU6ISV5EE/XAC7klxNYCMQCSd0NyqWCO16VCWHQ8hMx9wSG8fW+IQwEPNMQPlcGXwDlDksUMQDUpg59I2481fe0=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC10dW5pbmciLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNzMwMzdhYTY5NDE0NmNlNWRjNDhjMmEwOGNiYjIyYWQ5ZjllZDMwMTBkYmFhNWMxY2Q2MGNkNWNkMDg4NzIyYyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYjg5OWUxN2Y3OGM5Y2NmZjgzNDRlMmNlZmQxYTdmYTU2OGQ5ZjQ4NmJiNzYwYTRhZjUxOGRiMzQ2NDMxMDE3NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxNjQwMWRiY2Y5OTM0ZTY1MTkyNDdhNDNiN2VkYTYxNzc5NjRiMTJmNzBiNjg2MTY0MWU0NDE2MGU2ZDhlN2FiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNThiNmNmMzIyNDI1M2VlZTVmOWMxZTAzYjFmODJlM2E5NWVlOGYxYzUyN2MyNWNiYjM1YzE3NTQ4OTBkZGZlYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcGxhbnMubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjNlYjQyYjBmODk1M2QxNzAwMjU1OGQwNWE4ODI4NTE5MzFkOGNiNTEwY2E4ZjdlNWM2MGViOWNmNjc5MDJjNGQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkNjA1YWM0ZmYwOWExM2JiNDI4ZDRlNjE1NzNjYzA5MjQzMDE3N2UxZGY5YThiMWVkODYyNzc5OTExZDkyOWNhIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIKICAgICAgXSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCZ6bMenexf57YpQvEo5xKzIBIHwLJe7H49gkCHN8uVLJrQy5reHmerjfzhtZvX2X0CMQD4aD59e9Sq939kwAwMV+tYdFN/beIui0prh/cCDT21Dvih64JSH7FlBTHWvqfKF1o=","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-training-diagnostics/BENCHMARK.md b/skills/kumo-training-diagnostics/BENCHMARK.md index 736a075a0..4a5a316a6 100644 --- a/skills/kumo-training-diagnostics/BENCHMARK.md +++ b/skills/kumo-training-diagnostics/BENCHMARK.md @@ -11,7 +11,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-training-diagnostics` - Validation status: `passed` - Evaluation date: 2026-11-18 -- Evaluator version: `1.6.9` +- Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 5 evaluation tasks (4 positive, 1 negative) - Dataset digest: `sha256:fb29a11af5d6eacb6479640b84fd298f7b83d35cd2c861416fde9b4a96dc23f0` (skill-evaluator-dataset-snapshot/1) @@ -28,11 +28,11 @@ SkillSpector provider, model, analyzer revision, and determinism evidence retain | Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | | --- | --- | --- | --- | --- | --- | --- | -| kumo-training-diagnostics | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-training-diagnostics | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-training-diagnostics | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-training-diagnostics | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumo-training-diagnostics | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumo-training-diagnostics | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-training-diagnostics | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-training-diagnostics | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-training-diagnostics | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumo-training-diagnostics | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | - Controls for `kumo-training-diagnostics`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.9% — baseline ran, but no comparable score was available; uplift unavailable | 88.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 78.2% — baseline ran, but no comparable score was available; uplift unavailable | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | -| Correctness | 44.0% → 96.0% (+52.0 points) | 40.0% → 92.0% (+52.0 points) | -| Discoverability | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 48.2% → 58.5% (+10.3 points) | 50.2% → 63.2% (+13.0 points) | -| Efficiency | 95.1% — baseline ran, but no comparable score was available; uplift unavailable | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 48.0% → 80.0% (+32.0 points) | 32.0% → 88.0% (+56.0 points) | +| Discoverability | 75.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 38.2% → 45.7% (+7.5 points) | 36.8% → 65.2% (+28.4 points) | +| Efficiency | 90.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.4% → 96.5% (-1.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.5% → 96.9% (-1.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 92.2% → 91.8% (-0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.9% → 91.9% (+1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 686,834 | 574,054 | +112,780 | +19.65% | skill 5/5; base 5/5 | -| claude-code | kdiag-01 | 197,505 | 107,243 | +90,262 | +84.17% | skill 1/1; base 1/1 | -| claude-code | kdiag-02 | 155,071 | 107,205 | +47,866 | +44.65% | skill 1/1; base 1/1 | -| claude-code | kdiag-03 | 73,175 | 144,125 | -70,950 | -49.23% | skill 1/1; base 1/1 | -| claude-code | kdiag-04 | 153,506 | 144,150 | +9,356 | +6.49% | skill 1/1; base 1/1 | -| claude-code | kdiag-05 | 107,577 | 71,331 | +36,246 | +50.81% | skill 1/1; base 1/1 | -| codex | All cases | 306,969 | 221,991 | +84,978 | +38.28% | skill 5/5; base 5/5 | -| codex | kdiag-01 | 61,333 | 40,873 | +20,460 | +50.06% | skill 1/1; base 1/1 | -| codex | kdiag-02 | 79,718 | 63,335 | +16,383 | +25.87% | skill 1/1; base 1/1 | -| codex | kdiag-03 | 61,395 | 42,697 | +18,698 | +43.79% | skill 1/1; base 1/1 | -| codex | kdiag-04 | 59,414 | 57,295 | +2,119 | +3.70% | skill 1/1; base 1/1 | -| codex | kdiag-05 | 45,109 | 17,791 | +27,318 | +153.55% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 993,803 | 796,045 | +197,758 | +24.84% | skill 10/10; base 10/10 | +| claude-code | All cases | 715,359 | 535,886 | +179,473 | +33.49% | skill 5/5; base 5/5 | +| claude-code | kdiag-01 | 155,528 | 142,961 | +12,567 | +8.79% | skill 1/1; base 1/1 | +| claude-code | kdiag-02 | 197,199 | 71,171 | +126,028 | +177.08% | skill 1/1; base 1/1 | +| claude-code | kdiag-03 | 146,734 | 107,060 | +39,674 | +37.06% | skill 1/1; base 1/1 | +| claude-code | kdiag-04 | 108,287 | 143,366 | -35,079 | -24.47% | skill 1/1; base 1/1 | +| claude-code | kdiag-05 | 107,611 | 71,328 | +36,283 | +50.87% | skill 1/1; base 1/1 | +| codex | All cases | 291,440 | 252,141 | +39,299 | +15.59% | skill 5/5; base 5/5 | +| codex | kdiag-01 | 61,309 | 57,259 | +4,050 | +7.07% | skill 1/1; base 1/1 | +| codex | kdiag-02 | 78,558 | 77,086 | +1,472 | +1.91% | skill 1/1; base 1/1 | +| codex | kdiag-03 | 61,614 | 54,765 | +6,849 | +12.51% | skill 1/1; base 1/1 | +| codex | kdiag-04 | 44,357 | 45,264 | -907 | -2.00% | skill 1/1; base 1/1 | +| codex | kdiag-05 | 45,602 | 17,767 | +27,835 | +156.67% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,006,799 | 788,027 | +218,772 | +27.76% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-training-diagnostics/skill-card.md b/skills/kumo-training-diagnostics/skill-card.md index 7ead1f48f..7c5f6ea09 100644 --- a/skills/kumo-training-diagnostics/skill-card.md +++ b/skills/kumo-training-diagnostics/skill-card.md @@ -42,7 +42,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated `k8s-sandbox` pod; each task was also run without the skill as a baseline. Evaluator version `1.6.9`, evaluated 2026-11-18.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated `k8s-sandbox` pod; each task was also run without the skill as a baseline. Evaluator version `1.6.10`, evaluated 2026-11-18.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -74,12 +74,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.9% — baseline ran, but no comparable score was available; uplift unavailable | 88.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 78.2% — baseline ran, but no comparable score was available; uplift unavailable | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | -| Correctness | 44.0% → 96.0% (+52.0 points) | 40.0% → 92.0% (+52.0 points) | -| Discoverability | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 48.2% → 58.5% (+10.3 points) | 50.2% → 63.2% (+13.0 points) | -| Efficiency | 95.1% — baseline ran, but no comparable score was available; uplift unavailable | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 48.0% → 80.0% (+32.0 points) | 32.0% → 88.0% (+56.0 points) | +| Discoverability | 75.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 38.2% → 45.7% (+7.5 points) | 36.8% → 65.2% (+28.4 points) | +| Efficiency | 90.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -94,9 +94,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.4% → 96.5% (-1.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.5% → 96.9% (-1.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 92.2% → 91.8% (-0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.9% → 91.9% (+1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -107,7 +107,7 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | ## Skill Version(s):
-c7554e32 (source: git SHA, committed 2026-11-17)
+7cbda7af (source: git SHA, committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-training-diagnostics/skill.oms.sig b/skills/kumo-training-diagnostics/skill.oms.sig index 6417a08aa..e3b00f15b 100644 --- a/skills/kumo-training-diagnostics/skill.oms.sig +++ b/skills/kumo-training-diagnostics/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby10cmFpbmluZy1kaWFnbm9zdGljcyIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJiZTVkNmZlNmVlMDQzODc5ZTAzMWY0Y2JlOTA5NzY0NmU5YmEzNWFkYzZjYjc0NmI3MGUwNGFmNDY2MjJkNmIxIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJmZmIzM2FhYmRjZWM5YmRhZDJhODVmODEwMGUxZTUzM2VjZDMyNTkyZTFhOWI5Yzg1OWNiNTcyMDU5OWYzMDE4IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjY0ZGQ4OGFlNjYwMmZiMWYyN2E0ZjQzYTRlNDJkNjA3ZjczZmQ5ZjE3YWE0ODgxNWEzOGNlOWY0MzQwZmVmNzYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICI4YzRjN2IzYTY5ZGQxNDcxYTAzYjI1NzNiN2JlZTE5NDExMjliN2E1MmM4OTk4ZWI4OTc3MjJkZjIxNTUyOTU0IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvZGlhZ25vc3RpY3MubWQiLAogICAgICAgICJkaWdlc3QiOiAiZTc1ZDMwZTU0NTUzMjJmODVjNTQwZDhiNDM3ZGIwYzAwNzYxZDUyNzU4YTUyZjNiOWE4YTc2ODAzNjFmY2UzMCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjkwMjE3MDllNWQ2NmYyMGZlMTM5MGVkOTM5ZTY3NmVjNWIyZjIwNTJiMDQ2Y2IzN2RmZDkwNmRiNzYzOGNjM2YiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDYqhjTNeHin6sfGPm8DpoxH4SAhTU/6p0Kgwe3FnUWxxDPZ+DZAXlT/PGvgqtSOSUCMECB5Kq6SNSlJQapSZ1A8Vyq07vSJ+3PQQtKgYujqftcqPS/wbLqnJiSGAgZhdZhzw==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby10cmFpbmluZy1kaWFnbm9zdGljcyIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI3MGQ5YWJmOWQyMGU4YzBmYjYzYmFlNmI4MWMyYThlYjMzZWFmMzg0YWI4MjUwNDVjZTkxOGNjYWMxY2VhMGJlIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxZWYwMWYzM2E2ZDhmNWEzM2NiNDIyZTc5NzNkYzM2NWM3OWU5YjI1YzkzMTFiYjFjYzAwZjVlYzhkZjQ5YzYxIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjY0ZGQ4OGFlNjYwMmZiMWYyN2E0ZjQzYTRlNDJkNjA3ZjczZmQ5ZjE3YWE0ODgxNWEzOGNlOWY0MzQwZmVmNzYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI4YzRjN2IzYTY5ZGQxNDcxYTAzYjI1NzNiN2JlZTE5NDExMjliN2E1MmM4OTk4ZWI4OTc3MjJkZjIxNTUyOTU0IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9kaWFnbm9zdGljcy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZTc1ZDMwZTU0NTUzMjJmODVjNTQwZDhiNDM3ZGIwYzAwNzYxZDUyNzU4YTUyZjNiOWE4YTc2ODAzNjFmY2UzMCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjlhMjllMzE3NzM5ODU2OTQxZDczNzI3MzE5YTQzNmI2YjBhYWRkYTNjNDk3MjAwYWI5N2E1OGZkZGUyZGJhZTMiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIKICAgICAgXSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMC43nABz4HhURsMYsNFBWXibb9nEu1EYKoWd1qD0up+Fe2KTQWbMoiPp60kpy7OW+AIxAOfCkMC7Zyenk3Yty7yK8IJZGT4bi7hxoWVGzhkVBciwg6t4305PksMMU59M/gv0AQ==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumoai-reference/BENCHMARK.md b/skills/kumoai-reference/BENCHMARK.md index b8bdcde49..db1b6eedd 100644 --- a/skills/kumoai-reference/BENCHMARK.md +++ b/skills/kumoai-reference/BENCHMARK.md @@ -11,7 +11,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumoai-reference` - Validation status: `passed` - Evaluation date: 2026-11-18 -- Evaluator version: `1.6.9` +- Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 5 evaluation tasks (4 positive, 1 negative) - Dataset digest: `sha256:70c6edab12642418c73b37fd93ff86ac13f4ce8f782eac415e5aa695a3c59505` (skill-evaluator-dataset-snapshot/1) @@ -28,11 +28,11 @@ SkillSpector provider, model, analyzer revision, and determinism evidence retain | Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | | --- | --- | --- | --- | --- | --- | --- | -| kumoai-reference | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumoai-reference | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumoai-reference | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumoai-reference | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | -| kumoai-reference | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| kumoai-reference | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumoai-reference | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumoai-reference | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumoai-reference | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | +| kumoai-reference | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.5` | nondeterministic / provider_defaults | - Controls for `kumoai-reference`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 94.7% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 80.0% → 100.0% (+20.0 points) | 90.0% → 100.0% (+10.0 points) | -| Correctness | 64.0% → 100.0% (+36.0 points) | 96.0% → 100.0% (+4.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 50.2% → 80.7% (+30.5 points) | 68.9% → 91.6% (+22.7 points) | -| Efficiency | 87.7% — baseline ran, but no comparable score was available; uplift unavailable | 88.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 93.6% — baseline ran, but no comparable score was available; uplift unavailable | 89.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 80.0% → 100.0% (+20.0 points) | 90.0% → 90.0% (±0.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 92.0% → 100.0% (+8.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 44.9% → 84.0% (+39.1 points) | 77.8% → 80.4% (+2.6 points) | +| Efficiency | 83.8% — baseline ran, but no comparable score was available; uplift unavailable | 84.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 92.0% → 88.6% (-3.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.2% → 81.5% (-15.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 80.8% → 86.5% (+5.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 77.6% → 87.0% (+9.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,201,501 | 1,365,680 | -164,179 | -12.02% | skill 5/5; base 5/5 | -| claude-code | kref-01 | 121,639 | 35,557 | +86,082 | +242.10% | skill 1/1; base 1/1 | -| claude-code | kref-02 | 380,338 | 187,276 | +193,062 | +103.09% | skill 1/1; base 1/1 | -| claude-code | kref-03 | 233,155 | 549,777 | -316,622 | -57.59% | skill 1/1; base 1/1 | -| claude-code | kref-04 | 318,106 | 109,559 | +208,547 | +190.35% | skill 1/1; base 1/1 | -| claude-code | kref-05 | 148,263 | 483,511 | -335,248 | -69.34% | skill 1/1; base 1/1 | -| codex | All cases | 915,158 | 605,989 | +309,169 | +51.02% | skill 5/5; base 5/5 | -| codex | kref-01 | 64,938 | 46,310 | +18,628 | +40.22% | skill 1/1; base 1/1 | -| codex | kref-02 | 188,313 | 249,632 | -61,319 | -24.56% | skill 1/1; base 1/1 | -| codex | kref-03 | 135,073 | 75,281 | +59,792 | +79.43% | skill 1/1; base 1/1 | -| codex | kref-04 | 140,779 | 51,536 | +89,243 | +173.17% | skill 1/1; base 1/1 | -| codex | kref-05 | 386,055 | 183,230 | +202,825 | +110.69% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 2,116,659 | 1,971,669 | +144,990 | +7.35% | skill 10/10; base 10/10 | +| claude-code | All cases | 1,477,453 | 598,733 | +878,720 | +146.76% | skill 5/5; base 5/5 | +| claude-code | kref-01 | 121,691 | 35,737 | +85,954 | +240.52% | skill 1/1; base 1/1 | +| claude-code | kref-02 | 445,404 | 148,446 | +296,958 | +200.04% | skill 1/1; base 1/1 | +| claude-code | kref-03 | 134,125 | 108,917 | +25,208 | +23.14% | skill 1/1; base 1/1 | +| claude-code | kref-04 | 250,746 | 38,242 | +212,504 | +555.68% | skill 1/1; base 1/1 | +| claude-code | kref-05 | 525,487 | 267,391 | +258,096 | +96.52% | skill 1/1; base 1/1 | +| codex | All cases | 751,517 | 759,151 | -7,634 | -1.01% | skill 5/5; base 5/5 | +| codex | kref-01 | 65,056 | 54,410 | +10,646 | +19.57% | skill 1/1; base 1/1 | +| codex | kref-02 | 183,823 | 190,612 | -6,789 | -3.56% | skill 1/1; base 1/1 | +| codex | kref-03 | 121,056 | 90,774 | +30,282 | +33.36% | skill 1/1; base 1/1 | +| codex | kref-04 | 142,837 | 47,313 | +95,524 | +201.90% | skill 1/1; base 1/1 | +| codex | kref-05 | 238,745 | 376,042 | -137,297 | -36.51% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,228,970 | 1,357,884 | +871,086 | +64.15% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -100,7 +100,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 15 finding(s) | | Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 2 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | @@ -109,19 +109,19 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c
Show detailed findings and successful checks -- **HIGH** DUPLICATE/duplicate: Duplicate content found across references/graph_construction.md and references/kumo_finetuning_sdk.md: - "### Defining tables" in references/graph_construction.md (lines 106-143) - vs "## 4. Tables" in references/kumo_finetuning_sdk.md (lines 308-354) (`references/graph_construction.md:106`) - **HIGH** DUPLICATE/duplicate: Duplicate content found across references/kumo_finetuning_sdk.md and references/pql_writing.md and references/task_journeys.md: "## 6. Predictive query" in references/kumo_finetuning_sdk.md (lines 393-414) vs "## Task type is inferred from query shape" in references/pql_writing.md (lines 202-220) vs "## SDK usage" in references/pql_writing.md (lines 438-471) vs "### Required pre-flight, before any training step" in references/pql_writing.md (lines 472-490) vs "## Classification and regression" in references/task_journeys.md (lines 9-26) (`references/kumo_finetuning_sdk.md:393`) +- **HIGH** DUPLICATE/duplicate: Duplicate content found across references/graph_construction.md and references/kumo_finetuning_sdk.md: + "### Defining tables" in references/graph_construction.md (lines 106-143) + vs "## 4. Tables" in references/kumo_finetuning_sdk.md (lines 308-354) (`references/graph_construction.md:106`) - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumoai-reference/SKILL.md`) - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/kumoai-reference/SKILL.md`) - **MEDIUM** QUALITY/quality_efficiency: Deeply nested references in outputs.md (`skills/kumoai-reference/SKILL.md`) -- 11 additional finding(s) are available in the full evaluation artifacts. +- 12 additional finding(s) are available in the full evaluation artifacts.
diff --git a/skills/kumoai-reference/skill-card.md b/skills/kumoai-reference/skill-card.md index 97495a8e1..068961f4c 100644 --- a/skills/kumoai-reference/skill-card.md +++ b/skills/kumoai-reference/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Reference for translating a validated ML request into KumoAI graph configuration, PQL (predictive query language) and Kumo Predict SDK code; normally loaded by the kumo-data-science skill after its intake checks pass.
+Reference for translating a validated ML request into KumoAI graph configuration, PQL (predictive query language), and Kumo Predict SDK code; normally loaded by the kumo-data-science skill after its intake checks pass, rather than invoked directly.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill, normally through the kumo-data-science skill, so a coding agent writes Kumo graph definitions, predictive queries and training and prediction code that match the shipped Kumo SDK.
+Developers and data scientists use this skill, normally through the kumo-data-science skill, so a coding agent writes Kumo graph definitions, predictive queries, and training and prediction code that match the shipped Kumo SDK.
### Deployment Geography for Use:
Global
@@ -39,7 +39,7 @@ Mitigation: Review and scan skill before deployment.
**Output Type(s):** [Code, Analysis]
**Output Format:** [Markdown with inline Python and PQL code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Generates code only; does not run jobs, interpret metrics or repair runtime errors.]
+**Other Properties Related to Output:** [Generates code only; does not run jobs, interpret metrics, or repair runtime errors.]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -48,7 +48,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.9`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.10`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -80,12 +80,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 94.7% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 80.0% → 100.0% (+20.0 points) | 90.0% → 100.0% (+10.0 points) | -| Correctness | 64.0% → 100.0% (+36.0 points) | 96.0% → 100.0% (+4.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 50.2% → 80.7% (+30.5 points) | 68.9% → 91.6% (+22.7 points) | -| Efficiency | 87.7% — baseline ran, but no comparable score was available; uplift unavailable | 88.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 93.6% — baseline ran, but no comparable score was available; uplift unavailable | 89.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 80.0% → 100.0% (+20.0 points) | 90.0% → 90.0% (±0.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 92.0% → 100.0% (+8.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 44.9% → 84.0% (+39.1 points) | 77.8% → 80.4% (+2.6 points) | +| Efficiency | 83.8% — baseline ran, but no comparable score was available; uplift unavailable | 84.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -100,21 +100,20 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 92.0% → 88.6% (-3.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.2% → 81.5% (-15.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 80.8% → 86.5% (+5.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 77.6% → 87.0% (+9.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 15 finding(s) | | Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 2 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | - ## Skill Version(s):
-c7554e32 (source: git SHA, committed 2026-11-17); repository pyproject.toml version 0.1.0
+0.1.0 (source: pyproject.toml; git SHA 7cbda7af, committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumoai-reference/skill.oms.sig b/skills/kumoai-reference/skill.oms.sig index 690dd829f..e3a35edbd 100644 --- a/skills/kumoai-reference/skill.oms.sig +++ b/skills/kumoai-reference/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtb2FpLXJlZmVyZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJmODI4ZmY1OWM3ZjhhZGJiYTQ2ZmM0MTcxNTEzOTMwNTEyOWM5YmFlNzI5MThiZjhmYzNkODUxY2ExNzM1MzkyIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZjQwZDU1NDI4MzJlMTYyZDNjMzE3MjcyMDk3YzVlN2I4ZTEzYTJlNDIxZDgxYWE1OTE4OTUwZTk3ZTMxNDViOCIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMDQzMTYyYmFiYzc1NWFiYTEzNTgyNWFmM2NhMzc1ZDhmMGQ1MWQ5MDRjZGNhYmU2M2Y3MzlhNjNmODRjZDRhZiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlODBjYWNhY2NjNjA3YjIwMDFmYzE5MDQ0ZGNkNzEwOTJiNWEwNTQzYmY1NzYzMDJkZmRmMDgxMmY0NzhkZTZjIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOGU4OGUyM2IxN2EyMGUxN2E5Mjc4Njg3NDJmYzQwNTA2NGE4OGQxNDVmMmNiODgzNmE4NTJlMzQwMDVlNGJiYyIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9ncmFwaF9jb25zdHJ1Y3Rpb24ubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlMDQxYWY1ZTkyZmI2NmYyMjE1NWU0NTk0Mjk5NmFlZjM4ZWFjNGE0Y2E3ZDE0YjlkOWNhZWU4ZWFmNmI0OWI4IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2t1bW9fZmluZXR1bmluZ19zZGsubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJmMjlmM2QyZDJiY2E5NTUwZGU1MjViYjViYzYyZTU1ODc4ZTBmNTRiYTdmNWYxMDliMDJmYzk3OGJiYjg1ZWJhIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL291dHB1dHMubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI0NTc4N2RhY2I4ZTk3NzJhZTk1YjA1NzRmMjk3ODM0ODdlNGNjYzliNzBhZWUxYzkwNGQ4MGVkMmU4NWMxM2FmIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BxbF93cml0aW5nLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzA5YzIzYmExMGM3YWEwMDg5OTY0NjAwMGExNmVkYjMyYzg5ZDI5YzYzNTllNjE0ZGM5YWM2YTJlYTkyMGYwOCIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy90YXNrX2pvdXJuZXlzLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNzliNTU1ZGJmMmQ0N2MyOThlY2U2NGY0ZjVkODg0ZGU3N2Q0Zjc2YTY1M2JhOWUyMTgzMzBjM2YxYzE0N2ZiNSIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy90cm91Ymxlc2hvb3Rpbmdfc2lnbmFscy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjliYjFhYzFhYjE0N2ZmZDc0ZDIzMTBhMTRmODkyZTExNTAyMWMxOWM2MzIxNWVmNGY3MzIzZDMyMzg2MDIxNGEiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMH371JMAol5RWnITIr3SP2pAH+WgIt8OcP/IYb+/IwCXBCEkNfQkLADXYfRqfW/uswIwOIF3gjisxrvA3TJC5bH4gMutJpyOIl4DGnE5eL/9U/aeiFDx1a3XPrsiuXdATIhi","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtb2FpLXJlZmVyZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJhY2YzZDk3ZDdlNWZlYWVjMTlhYjJlMzFjYmJjNjgxYzUzMzVlZDVkODczNDdkMmVkMGFjN2I5NWZkYjIxYmQ3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGh1YiIKICAgICAgXSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiNmIxMzhmMjg2NmJjNDU3MjVjNzNkZmUxODdlMmU2YmJiZjQ0NDE0MDU5ODMwMGQ5MDAyNGMwNGYxZWUzMDAzYSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIwNDMxNjJiYWJjNzU1YWJhMTM1ODI1YWYzY2EzNzVkOGYwZDUxZDkwNGNkY2FiZTYzZjczOWE2M2Y4NGNkNGFmIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiZTgwY2FjYWNjYzYwN2IyMDAxZmMxOTA0NGRjZDcxMDkyYjVhMDU0M2JmNTc2MzAyZGZkZjA4MTJmNDc4ZGU2YyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2dyYXBoX2NvbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI4ZTg4ZTIzYjE3YTIwZTE3YTkyNzg2ODc0MmZjNDA1MDY0YTg4ZDE0NWYyY2I4ODM2YTg1MmUzNDAwNWU0YmJjIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMva3Vtb19maW5ldHVuaW5nX3Nkay5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJlMDQxYWY1ZTkyZmI2NmYyMjE1NWU0NTk0Mjk5NmFlZjM4ZWFjNGE0Y2E3ZDE0YjlkOWNhZWU4ZWFmNmI0OWI4IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvb3V0cHV0cy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJmMjlmM2QyZDJiY2E5NTUwZGU1MjViYjViYzYyZTU1ODc4ZTBmNTRiYTdmNWYxMDliMDJmYzk3OGJiYjg1ZWJhIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcHFsX3dyaXRpbmcubWQiLAogICAgICAgICJkaWdlc3QiOiAiNDU3ODdkYWNiOGU5NzcyYWU5NWIwNTc0ZjI5NzgzNDg3ZTRjY2M5YjcwYWVlMWM5MDRkODBlZDJlODVjMTNhZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3Rhc2tfam91cm5leXMubWQiLAogICAgICAgICJkaWdlc3QiOiAiYzA5YzIzYmExMGM3YWEwMDg5OTY0NjAwMGExNmVkYjMyYzg5ZDI5YzYzNTllNjE0ZGM5YWM2YTJlYTkyMGYwOCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3Ryb3VibGVzaG9vdGluZ19zaWduYWxzLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjc5YjU1NWRiZjJkNDdjMjk4ZWNlNjRmNGY1ZDg4NGRlNzdkNGY3NmE2NTNiYTllMjE4MzMwYzNmMWMxNDdmYjUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJhNDBiYTQ3NDBmOWU2ZThlMTA5MTEwMjFmZGQ4NDBiMTc4NzdlYjU0NzUwZGUwNWUzMWVmYTBhZTMxZjgzNWRjIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQD1KW//rd8bZMnnOxeBxX4wR/iDo/3dejv0QP21FzIC2FXpIXjKg8q0iZjTqZ5xetYCMQCDq5Z6IQ3T/bxSRXcEGAjw49Bkuqn+y5/I2WoOzIbKs62b8ptNxYJ2WFsIwV2An0Y=","keyid":""}]}} \ No newline at end of file From 83ce9a89af76c1926443d6c6f57b7e87faff6e25 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Wed, 18 Nov 2026 13:01:29 -0600 Subject: [PATCH 29/35] agent-skills: stage a population time column in its declared format first With the cast first, text that also parses as ISO ignored the entity's timestamp_format: `2024-01-02` under `%Y-%d-%m` staged as January 2 while the graph reads February 1. A CSV time column with a declared format is now read as text, parsed with the format first, and cast only when Arrow cannot read a chrono directive such as %.f. Co-Authored-By: Claude Opus 5.5 --- agent-skills/skills/hosts/generic/runtime.md | 33 +++++++++++-------- skills/kumo-data-science/SKILL.md | 2 +- .../hosts/generic/runtime.md | 33 +++++++++++-------- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/agent-skills/skills/hosts/generic/runtime.md b/agent-skills/skills/hosts/generic/runtime.md index 18c05cae3..683034bb9 100644 --- a/agent-skills/skills/hosts/generic/runtime.md +++ b/agent-skills/skills/hosts/generic/runtime.md @@ -317,6 +317,7 @@ system; the destination may be in another system (`adapter.md` §4) and plays no import os import pyarrow as pa import pyarrow.compute as pc + import pyarrow.csv import pyarrow.dataset as ds import pyarrow.fs import pyarrow.parquet as papq @@ -333,7 +334,11 @@ system; the destination may be in another system (`adapter.md` §4) and plays no if ENTITY_TIME: # the entity Table's time_column, else None keep[ENTITY_TIME] = 'TIMESTAMP' source_fs, source_path = arrow_fs(ENTITY_SOURCE) - scanner = ds.dataset(source_path, format=ENTITY_FORMAT, + source_format = ENTITY_FORMAT + if ENTITY_FORMAT == 'csv' and ENTITY_TIME and ENTITY_TIME_FORMAT: # text, for the format below + source_format = ds.CsvFileFormat(convert_options=pyarrow.csv.ConvertOptions( + column_types={ENTITY_TIME: pa.string()})) + scanner = ds.dataset(source_path, format=source_format, filesystem=source_fs).scanner(columns=list(keep)) def anchor_type(t): # a timestamp keeps its unit; a DATE anchor reads as null: cast it @@ -342,13 +347,14 @@ system; the destination may be in another system (`adapter.md` §4) and plays no schema = pa.schema([(keep[f.name], anchor_type(f.type) if keep[f.name] == 'TIMESTAMP' else f.type) for f in scanner.projected_schema]) - def to_column(col, field): # a cast reads ISO-8601 text; other text needs the entity's format - try: - return pc.cast(col, field.type) - except pa.ArrowInvalid: - if not (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT): - raise - return pc.strptime(col, format=ENTITY_TIME_FORMAT, unit=field.type.unit) + def to_column(col, field): # text in the entity's format reads as the graph reads it + if (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT + and (pa.types.is_string(col.type) or pa.types.is_large_string(col.type))): + try: + return pc.strptime(col, format=ENTITY_TIME_FORMAT, unit=field.type.unit) + except pa.ArrowInvalid: + pass # a chrono directive Arrow cannot read, e.g. %.f: cast instead + return pc.cast(col, field.type) pop_fs, pop_dir = arrow_fs(POPULATION_PATH) if isinstance(pop_fs, pyarrow.fs.LocalFileSystem): @@ -375,11 +381,12 @@ system; the destination may be in another system (`adapter.md` §4) and plays no `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. `ENTITY_TIME_FORMAT` is the `timestamp_format` the entity `Table` declares on its time column (`kumoai-reference/references/graph_construction.md`, *A time column must be a real date*), else - `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The cast runs - first, so ISO text staged as before, fractional seconds included; only text it cannot read is - parsed with the format. That format is Kumo's (chrono) syntax, and Arrow's parser rejects - `%.f`, `%-m` and `%:z`: if the parse then fails, report the format and the failing value rather - than guessing another format. The population is + `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. Text is parsed + with the format first, because ISO-looking text such as `2024-01-02` under `%Y-%d-%m` means what + the format says, as it does in the graph. That format is Kumo's (chrono) syntax, and Arrow's + parser rejects `%.f`, `%-m` and `%:z`; when it cannot parse, the column is cast instead, which + reads ISO-8601 text, fractional seconds included. If the cast then fails, report the format and + the failing value rather than guessing another format. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed it, both in the config cell. Each run replaces its `population.parquet`, as Node 1d says. The diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index 9f75f4136..6a38a6acb 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-17-5026ce47-dirty`** +**`SKILL-BUILD: 2026-11-18-7cbda7af-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/hosts/generic/runtime.md b/skills/kumo-data-science/hosts/generic/runtime.md index 18c05cae3..683034bb9 100644 --- a/skills/kumo-data-science/hosts/generic/runtime.md +++ b/skills/kumo-data-science/hosts/generic/runtime.md @@ -317,6 +317,7 @@ system; the destination may be in another system (`adapter.md` §4) and plays no import os import pyarrow as pa import pyarrow.compute as pc + import pyarrow.csv import pyarrow.dataset as ds import pyarrow.fs import pyarrow.parquet as papq @@ -333,7 +334,11 @@ system; the destination may be in another system (`adapter.md` §4) and plays no if ENTITY_TIME: # the entity Table's time_column, else None keep[ENTITY_TIME] = 'TIMESTAMP' source_fs, source_path = arrow_fs(ENTITY_SOURCE) - scanner = ds.dataset(source_path, format=ENTITY_FORMAT, + source_format = ENTITY_FORMAT + if ENTITY_FORMAT == 'csv' and ENTITY_TIME and ENTITY_TIME_FORMAT: # text, for the format below + source_format = ds.CsvFileFormat(convert_options=pyarrow.csv.ConvertOptions( + column_types={ENTITY_TIME: pa.string()})) + scanner = ds.dataset(source_path, format=source_format, filesystem=source_fs).scanner(columns=list(keep)) def anchor_type(t): # a timestamp keeps its unit; a DATE anchor reads as null: cast it @@ -342,13 +347,14 @@ system; the destination may be in another system (`adapter.md` §4) and plays no schema = pa.schema([(keep[f.name], anchor_type(f.type) if keep[f.name] == 'TIMESTAMP' else f.type) for f in scanner.projected_schema]) - def to_column(col, field): # a cast reads ISO-8601 text; other text needs the entity's format - try: - return pc.cast(col, field.type) - except pa.ArrowInvalid: - if not (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT): - raise - return pc.strptime(col, format=ENTITY_TIME_FORMAT, unit=field.type.unit) + def to_column(col, field): # text in the entity's format reads as the graph reads it + if (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT + and (pa.types.is_string(col.type) or pa.types.is_large_string(col.type))): + try: + return pc.strptime(col, format=ENTITY_TIME_FORMAT, unit=field.type.unit) + except pa.ArrowInvalid: + pass # a chrono directive Arrow cannot read, e.g. %.f: cast instead + return pc.cast(col, field.type) pop_fs, pop_dir = arrow_fs(POPULATION_PATH) if isinstance(pop_fs, pyarrow.fs.LocalFileSystem): @@ -375,11 +381,12 @@ system; the destination may be in another system (`adapter.md` §4) and plays no `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. `ENTITY_TIME_FORMAT` is the `timestamp_format` the entity `Table` declares on its time column (`kumoai-reference/references/graph_construction.md`, *A time column must be a real date*), else - `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The cast runs - first, so ISO text staged as before, fractional seconds included; only text it cannot read is - parsed with the format. That format is Kumo's (chrono) syntax, and Arrow's parser rejects - `%.f`, `%-m` and `%:z`: if the parse then fails, report the format and the failing value rather - than guessing another format. The population is + `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. Text is parsed + with the format first, because ISO-looking text such as `2024-01-02` under `%Y-%d-%m` means what + the format says, as it does in the graph. That format is Kumo's (chrono) syntax, and Arrow's + parser rejects `%.f`, `%-m` and `%:z`; when it cannot parse, the column is cast instead, which + reads ISO-8601 text, fractional seconds included. If the cast then fails, report the format and + the failing value rather than guessing another format. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed it, both in the config cell. Each run replaces its `population.parquet`, as Node 1d says. The From 8aaa6cb3bbfa6eb6ccd4186e18f1d6dedb5050f4 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Wed, 18 Nov 2026 15:16:43 -0600 Subject: [PATCH 30/35] agent-skills: ask the persistence evals for notebook code NVSkills CI scored kumo-model-persistence NEUTRAL: Effectiveness was 0.49 against a 0.50 bar (0.506 when it last passed). Three cases told the agent to train, publish or upload in a sandbox with no Kumo, Snowflake or saved model, so the agent correctly reported it could not, and the grader marked the expected behaviour as not carried out. The skill's job is to write that code, and the passing skills' cases ask for a notebook. Those three now ask for the notebook cells, with the same requirements on the code. Co-Authored-By: Claude Opus 5.5 --- .../kumo-model-persistence/evals/evals.json | 44 +++++++++---------- skills/kumo-data-science/SKILL.md | 2 +- .../kumo-model-persistence/evals/evals.json | 44 +++++++++---------- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/agent-skills/nvskills/kumo-model-persistence/evals/evals.json b/agent-skills/nvskills/kumo-model-persistence/evals/evals.json index 6dcdc4911..822a3db58 100644 --- a/agent-skills/nvskills/kumo-model-persistence/evals/evals.json +++ b/agent-skills/nvskills/kumo-model-persistence/evals/evals.json @@ -3,14 +3,14 @@ "evals": [ { "id": "kpersist-01", - "prompt": "Train the confirmed Kumo churn model (model_plan, training_table and snapshot are ready in my Snowflake notebook) and publish it to the model registry in ML.REGISTRY as CHURN_MODEL version V3.", - "expected_output": "Publishes during the fit: `trainer.fit(training_table, snapshot=snapshot, export_config=kumo.ModelOutputConfig(connector=registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V3')))` with a connector scoped to ML.REGISTRY, then checks `training_result.published_model` and raises if it is None. Uses `location`, not `path`, for Snowflake. Checks that V3 does not already exist first and does not overwrite an existing version.", + "prompt": "Write the cell for my Snowflake notebook that trains the confirmed Kumo churn model and publishes it to the model registry in ML.REGISTRY as CHURN_MODEL version V3. `model_plan`, `training_table` and `snapshot` are already defined in the notebook. I will run the cell myself.", + "expected_output": "Writes code that publishes during the fit: `trainer.fit(training_table, snapshot=snapshot, export_config=kumo.ModelOutputConfig(connector=registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V3')))` with a connector scoped to ML.REGISTRY, then checks `training_result.published_model` and raises if it is None. Uses `location`, not `path`, for Snowflake. Checks that V3 does not already exist first and does not overwrite an existing version.", "assertions": [ - "Passes a kumo.ModelOutputConfig as export_config to trainer.fit", - "Uses kumo.ModelRegistryLocation with model_name CHURN_MODEL and version V3", - "Uses location= rather than path= for the Snowflake registry", - "Checks training_result.published_model and treats None as a failure", - "Checks that version V3 does not already exist and does not overwrite an existing version" + "The code passes a kumo.ModelOutputConfig as export_config to trainer.fit", + "The code uses kumo.ModelRegistryLocation with model_name CHURN_MODEL and version V3", + "The code uses location= rather than path= for the Snowflake registry", + "The code checks training_result.published_model and treats None as a failure", + "The code checks that version V3 does not already exist and does not overwrite an existing version" ], "expected_skill": "kumo-model-persistence", "expected_script": null @@ -29,29 +29,29 @@ }, { "id": "kpersist-03", - "prompt": "Load version V2 of CHURN_MODEL from the Snowflake registry ML.REGISTRY and score our customers as of 2026-12-01. The graph-building code and query are in our training record for V2.", - "expected_output": "Restores with `kumo.Trainer.load_from_connector(registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V2'))`, rebuilds the graph from the V2 training record (not the live schema), creates a fresh `graph.snapshot()`, generates the prediction table with a naive `prediction_time` and `plan=loaded.training_table_generation_plan`, and predicts with `snapshot=` and an explicit binary threshold. No retraining.", + "prompt": "Write the notebook cells that load version V2 of CHURN_MODEL from the Snowflake registry ML.REGISTRY and score our customers as of 2026-12-01. The V2 training record's graph-building code is in `build_graph_v2()`, already defined in the notebook. I will run the cells myself.", + "expected_output": "Writes code that restores with `kumo.Trainer.load_from_connector(registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V2'))`, rebuilds the graph with the V2 training record's `build_graph_v2()` (not the live schema), creates a fresh `graph.snapshot()`, generates the prediction table with a naive `prediction_time` and `plan=loaded.training_table_generation_plan`, and predicts with `snapshot=` and an explicit binary threshold. No retraining.", "assertions": [ - "Loads with kumo.Trainer.load_from_connector and an explicit ModelRegistryLocation for version V2", - "Rebuilds the graph from the V2 training record rather than from the live schema or another version", - "Builds a new graph.snapshot() for scoring", - "Passes plan=loaded.training_table_generation_plan to generate_prediction_table", - "Passes snapshot= to predict and an explicit binary classification threshold", - "Does not call fit" + "The code loads with kumo.Trainer.load_from_connector and an explicit ModelRegistryLocation for version V2", + "The code rebuilds the graph from the V2 training record (build_graph_v2) rather than from the live schema or another version", + "The code builds a new graph.snapshot() for scoring", + "The code passes plan=loaded.training_table_generation_plan to generate_prediction_table", + "The code passes snapshot= to predict and an explicit binary classification threshold", + "The code does not call fit" ], "expected_skill": "kumo-model-persistence", "expected_script": null }, { "id": "kpersist-04", - "prompt": "Before this Snowflake notebook shuts down, keep my Kumo run's aggregate metrics and the per-trial validation results on the internal stage @ML.ARTIFACTS so I can look at them next week.", - "expected_output": "Selects explicit files from one run (not the whole scratch tree, which can hold source data and credentials), writes aggregate metrics to a new JSON file with allow_nan=False, builds an archive with the packaged `create_archive` helper from an explicit list of relative paths, uploads with `session.file.put(..., auto_compress=False, overwrite=False)` to a new run-specific prefix and checks exactly one `UPLOADED` result, and records the stage path and archive SHA256 digest.", + "prompt": "Write the cell for my Snowflake notebook that keeps my Kumo run's aggregate metrics and the per-trial validation results on the internal stage @ML.ARTIFACTS before the notebook shuts down, so I can look at them next week. The run's directory is `run_dir`, already defined. I will run the cell myself.", + "expected_output": "Writes code that selects explicit files from `run_dir` (not the whole scratch tree, which can hold source data and credentials), writes aggregate metrics to a new JSON file with allow_nan=False, builds an archive with the packaged `create_archive` helper from an explicit list of relative paths, uploads with `session.file.put(..., auto_compress=False, overwrite=False)` to a new run-specific prefix and checks exactly one `UPLOADED` result, and records the stage path and archive SHA256 digest.", "assertions": [ - "Selects explicit files from one run instead of uploading the whole scratch directory", - "Uses the packaged create_archive helper with a list of explicit relative file paths", - "Uploads with session.file.put and overwrite=False", - "Checks that the upload status is UPLOADED and treats anything else, including SKIPPED, as not retained", - "Records the stage object path and the archive digest" + "The code selects explicit files from one run instead of uploading the whole scratch directory", + "The code uses the packaged create_archive helper with a list of explicit relative file paths", + "The code uploads with session.file.put and overwrite=False", + "The code checks that the upload status is UPLOADED and treats anything else, including SKIPPED, as not retained", + "The code records the stage object path and the archive digest" ], "expected_skill": "kumo-model-persistence", "expected_script": null diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index 6a38a6acb..d2365482c 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-18-7cbda7af-dirty`** +**`SKILL-BUILD: 2026-11-18-83ce9a89`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-model-persistence/evals/evals.json b/skills/kumo-model-persistence/evals/evals.json index 6dcdc4911..822a3db58 100644 --- a/skills/kumo-model-persistence/evals/evals.json +++ b/skills/kumo-model-persistence/evals/evals.json @@ -3,14 +3,14 @@ "evals": [ { "id": "kpersist-01", - "prompt": "Train the confirmed Kumo churn model (model_plan, training_table and snapshot are ready in my Snowflake notebook) and publish it to the model registry in ML.REGISTRY as CHURN_MODEL version V3.", - "expected_output": "Publishes during the fit: `trainer.fit(training_table, snapshot=snapshot, export_config=kumo.ModelOutputConfig(connector=registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V3')))` with a connector scoped to ML.REGISTRY, then checks `training_result.published_model` and raises if it is None. Uses `location`, not `path`, for Snowflake. Checks that V3 does not already exist first and does not overwrite an existing version.", + "prompt": "Write the cell for my Snowflake notebook that trains the confirmed Kumo churn model and publishes it to the model registry in ML.REGISTRY as CHURN_MODEL version V3. `model_plan`, `training_table` and `snapshot` are already defined in the notebook. I will run the cell myself.", + "expected_output": "Writes code that publishes during the fit: `trainer.fit(training_table, snapshot=snapshot, export_config=kumo.ModelOutputConfig(connector=registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V3')))` with a connector scoped to ML.REGISTRY, then checks `training_result.published_model` and raises if it is None. Uses `location`, not `path`, for Snowflake. Checks that V3 does not already exist first and does not overwrite an existing version.", "assertions": [ - "Passes a kumo.ModelOutputConfig as export_config to trainer.fit", - "Uses kumo.ModelRegistryLocation with model_name CHURN_MODEL and version V3", - "Uses location= rather than path= for the Snowflake registry", - "Checks training_result.published_model and treats None as a failure", - "Checks that version V3 does not already exist and does not overwrite an existing version" + "The code passes a kumo.ModelOutputConfig as export_config to trainer.fit", + "The code uses kumo.ModelRegistryLocation with model_name CHURN_MODEL and version V3", + "The code uses location= rather than path= for the Snowflake registry", + "The code checks training_result.published_model and treats None as a failure", + "The code checks that version V3 does not already exist and does not overwrite an existing version" ], "expected_skill": "kumo-model-persistence", "expected_script": null @@ -29,29 +29,29 @@ }, { "id": "kpersist-03", - "prompt": "Load version V2 of CHURN_MODEL from the Snowflake registry ML.REGISTRY and score our customers as of 2026-12-01. The graph-building code and query are in our training record for V2.", - "expected_output": "Restores with `kumo.Trainer.load_from_connector(registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V2'))`, rebuilds the graph from the V2 training record (not the live schema), creates a fresh `graph.snapshot()`, generates the prediction table with a naive `prediction_time` and `plan=loaded.training_table_generation_plan`, and predicts with `snapshot=` and an explicit binary threshold. No retraining.", + "prompt": "Write the notebook cells that load version V2 of CHURN_MODEL from the Snowflake registry ML.REGISTRY and score our customers as of 2026-12-01. The V2 training record's graph-building code is in `build_graph_v2()`, already defined in the notebook. I will run the cells myself.", + "expected_output": "Writes code that restores with `kumo.Trainer.load_from_connector(registry_connector, location=kumo.ModelRegistryLocation(model_name='CHURN_MODEL', version='V2'))`, rebuilds the graph with the V2 training record's `build_graph_v2()` (not the live schema), creates a fresh `graph.snapshot()`, generates the prediction table with a naive `prediction_time` and `plan=loaded.training_table_generation_plan`, and predicts with `snapshot=` and an explicit binary threshold. No retraining.", "assertions": [ - "Loads with kumo.Trainer.load_from_connector and an explicit ModelRegistryLocation for version V2", - "Rebuilds the graph from the V2 training record rather than from the live schema or another version", - "Builds a new graph.snapshot() for scoring", - "Passes plan=loaded.training_table_generation_plan to generate_prediction_table", - "Passes snapshot= to predict and an explicit binary classification threshold", - "Does not call fit" + "The code loads with kumo.Trainer.load_from_connector and an explicit ModelRegistryLocation for version V2", + "The code rebuilds the graph from the V2 training record (build_graph_v2) rather than from the live schema or another version", + "The code builds a new graph.snapshot() for scoring", + "The code passes plan=loaded.training_table_generation_plan to generate_prediction_table", + "The code passes snapshot= to predict and an explicit binary classification threshold", + "The code does not call fit" ], "expected_skill": "kumo-model-persistence", "expected_script": null }, { "id": "kpersist-04", - "prompt": "Before this Snowflake notebook shuts down, keep my Kumo run's aggregate metrics and the per-trial validation results on the internal stage @ML.ARTIFACTS so I can look at them next week.", - "expected_output": "Selects explicit files from one run (not the whole scratch tree, which can hold source data and credentials), writes aggregate metrics to a new JSON file with allow_nan=False, builds an archive with the packaged `create_archive` helper from an explicit list of relative paths, uploads with `session.file.put(..., auto_compress=False, overwrite=False)` to a new run-specific prefix and checks exactly one `UPLOADED` result, and records the stage path and archive SHA256 digest.", + "prompt": "Write the cell for my Snowflake notebook that keeps my Kumo run's aggregate metrics and the per-trial validation results on the internal stage @ML.ARTIFACTS before the notebook shuts down, so I can look at them next week. The run's directory is `run_dir`, already defined. I will run the cell myself.", + "expected_output": "Writes code that selects explicit files from `run_dir` (not the whole scratch tree, which can hold source data and credentials), writes aggregate metrics to a new JSON file with allow_nan=False, builds an archive with the packaged `create_archive` helper from an explicit list of relative paths, uploads with `session.file.put(..., auto_compress=False, overwrite=False)` to a new run-specific prefix and checks exactly one `UPLOADED` result, and records the stage path and archive SHA256 digest.", "assertions": [ - "Selects explicit files from one run instead of uploading the whole scratch directory", - "Uses the packaged create_archive helper with a list of explicit relative file paths", - "Uploads with session.file.put and overwrite=False", - "Checks that the upload status is UPLOADED and treats anything else, including SKIPPED, as not retained", - "Records the stage object path and the archive digest" + "The code selects explicit files from one run instead of uploading the whole scratch directory", + "The code uses the packaged create_archive helper with a list of explicit relative file paths", + "The code uploads with session.file.put and overwrite=False", + "The code checks that the upload status is UPLOADED and treats anything else, including SKIPPED, as not retained", + "The code records the stage object path and the archive digest" ], "expected_skill": "kumo-model-persistence", "expected_script": null From 7218a192fc3979efc181801864f4b35de0ca15a4 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Wed, 18 Nov 2026 22:25:10 +0000 Subject: [PATCH 31/35] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/kumo-custom-populations/BENCHMARK.md | 34 +++++----- skills/kumo-custom-populations/skill-card.md | 10 +-- skills/kumo-custom-populations/skill.oms.sig | 2 +- skills/kumo-data-science/BENCHMARK.md | 62 +++++++++---------- skills/kumo-data-science/skill-card.md | 28 +++++---- skills/kumo-data-science/skill.oms.sig | 2 +- skills/kumo-graph-visualization/BENCHMARK.md | 40 ++++++------ skills/kumo-graph-visualization/skill-card.md | 56 ++++++++--------- skills/kumo-graph-visualization/skill.oms.sig | 2 +- skills/kumo-model-explanations/BENCHMARK.md | 42 ++++++------- skills/kumo-model-explanations/skill-card.md | 20 +++--- skills/kumo-model-explanations/skill.oms.sig | 2 +- skills/kumo-model-persistence/BENCHMARK.md | 48 +++++++------- skills/kumo-model-persistence/skill-card.md | 34 +++++----- skills/kumo-model-persistence/skill.oms.sig | 2 +- skills/kumo-model-tuning/BENCHMARK.md | 38 ++++++------ skills/kumo-model-tuning/skill-card.md | 18 +++--- skills/kumo-model-tuning/skill.oms.sig | 2 +- skills/kumo-training-diagnostics/BENCHMARK.md | 42 ++++++------- .../kumo-training-diagnostics/skill-card.md | 53 ++++++---------- .../kumo-training-diagnostics/skill.oms.sig | 2 +- skills/kumoai-reference/BENCHMARK.md | 42 ++++++------- skills/kumoai-reference/skill-card.md | 30 ++++----- skills/kumoai-reference/skill.oms.sig | 2 +- 24 files changed, 308 insertions(+), 305 deletions(-) diff --git a/skills/kumo-custom-populations/BENCHMARK.md b/skills/kumo-custom-populations/BENCHMARK.md index f049f04b7..d6301d187 100644 --- a/skills/kumo-custom-populations/BENCHMARK.md +++ b/skills/kumo-custom-populations/BENCHMARK.md @@ -52,10 +52,10 @@ The three-tier evaluation checks whether the skill: |---|---:|---:| | Overall | 95.1% — baseline ran, but no comparable score was available; uplift unavailable | 90.2% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 52.0% → 100.0% (+48.0 points) | 68.0% → 96.0% (+28.0 points) | +| Correctness | 48.0% → 100.0% (+52.0 points) | 64.0% → 92.0% (+28.0 points) | | Discoverability | 97.5% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 44.0% → 80.0% (+36.0 points) | 42.0% → 64.0% (+22.0 points) | -| Efficiency | 98.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 48.0% → 84.0% (+36.0 points) | 44.0% → 68.0% (+24.0 points) | +| Efficiency | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -72,7 +72,7 @@ Shared-source comparisons use the same signals and weights on both conditions. T | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | | claude-code | Efficiency | Token-only comparison | 98.0% → 96.2% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.0% → 92.2% (+3.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.4% → 92.1% (+2.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 694,555 | 651,222 | +43,333 | +6.65% | skill 5/5; base 5/5 | -| claude-code | kpop-01 | 153,733 | 146,041 | +7,692 | +5.27% | skill 1/1; base 1/1 | -| claude-code | kpop-02 | 195,226 | 144,634 | +50,592 | +34.98% | skill 1/1; base 1/1 | -| claude-code | kpop-03 | 154,848 | 106,828 | +48,020 | +44.95% | skill 1/1; base 1/1 | -| claude-code | kpop-04 | 155,043 | 218,081 | -63,038 | -28.91% | skill 1/1; base 1/1 | -| claude-code | kpop-05 | 35,705 | 35,638 | +67 | +0.19% | skill 1/1; base 1/1 | -| codex | All cases | 209,249 | 276,533 | -67,284 | -24.33% | skill 5/5; base 5/5 | -| codex | kpop-01 | 43,763 | 61,911 | -18,148 | -29.31% | skill 1/1; base 1/1 | -| codex | kpop-02 | 60,254 | 60,332 | -78 | -0.13% | skill 1/1; base 1/1 | -| codex | kpop-03 | 43,684 | 55,934 | -12,250 | -21.90% | skill 1/1; base 1/1 | -| codex | kpop-04 | 43,771 | 80,750 | -36,979 | -45.79% | skill 1/1; base 1/1 | -| codex | kpop-05 | 17,777 | 17,606 | +171 | +0.97% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 903,804 | 927,755 | -23,951 | -2.58% | skill 10/10; base 10/10 | +| claude-code | All cases | 612,687 | 579,473 | +33,214 | +5.73% | skill 5/5; base 5/5 | +| claude-code | kpop-01 | 154,399 | 110,856 | +43,543 | +39.28% | skill 1/1; base 1/1 | +| claude-code | kpop-02 | 153,205 | 143,636 | +9,569 | +6.66% | skill 1/1; base 1/1 | +| claude-code | kpop-03 | 155,016 | 107,009 | +48,007 | +44.86% | skill 1/1; base 1/1 | +| claude-code | kpop-04 | 114,314 | 182,168 | -67,854 | -37.25% | skill 1/1; base 1/1 | +| claude-code | kpop-05 | 35,753 | 35,804 | -51 | -0.14% | skill 1/1; base 1/1 | +| codex | All cases | 193,397 | 256,704 | -63,307 | -24.66% | skill 5/5; base 5/5 | +| codex | kpop-01 | 43,779 | 57,412 | -13,633 | -23.75% | skill 1/1; base 1/1 | +| codex | kpop-02 | 43,773 | 56,327 | -12,554 | -22.29% | skill 1/1; base 1/1 | +| codex | kpop-03 | 43,992 | 41,800 | +2,192 | +5.24% | skill 1/1; base 1/1 | +| codex | kpop-04 | 44,154 | 83,522 | -39,368 | -47.13% | skill 1/1; base 1/1 | +| codex | kpop-05 | 17,699 | 17,643 | +56 | +0.32% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 806,084 | 836,177 | -30,093 | -3.60% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-custom-populations/skill-card.md b/skills/kumo-custom-populations/skill-card.md index cf57d4da4..eee914a9e 100644 --- a/skills/kumo-custom-populations/skill-card.md +++ b/skills/kumo-custom-populations/skill-card.md @@ -76,10 +76,10 @@ Recommended for publication based on the completed evaluation evidence in this r |---|---:|---:| | Overall | 95.1% — baseline ran, but no comparable score was available; uplift unavailable | 90.2% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 52.0% → 100.0% (+48.0 points) | 68.0% → 96.0% (+28.0 points) | +| Correctness | 48.0% → 100.0% (+52.0 points) | 64.0% → 92.0% (+28.0 points) | | Discoverability | 97.5% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 44.0% → 80.0% (+36.0 points) | 42.0% → 64.0% (+22.0 points) | -| Efficiency | 98.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 48.0% → 84.0% (+36.0 points) | 44.0% → 68.0% (+24.0 points) | +| Efficiency | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -87,7 +87,7 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. -### Comparison details +#### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. @@ -96,7 +96,7 @@ Shared-source comparisons use the same signals and weights on both conditions. T | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | | claude-code | Efficiency | Token-only comparison | 98.0% → 96.2% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.0% → 92.2% (+3.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.4% → 92.1% (+2.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status diff --git a/skills/kumo-custom-populations/skill.oms.sig b/skills/kumo-custom-populations/skill.oms.sig index 1b319007e..013d21387 100644 --- a/skills/kumo-custom-populations/skill.oms.sig +++ b/skills/kumo-custom-populations/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1jdXN0b20tcG9wdWxhdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNTcyMWNkNDBjZDJhYjZjYmM3OTc5Y2ZiNDQwNzgxZDE1MDcyMGU4ZTQ3OGY5ZDgzZjIwMjMwNDUzZWRlZjIyMiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYThjYzk1MmVhNjc2ZWUyNjk2ODRiZWZjMjExYjMwMzMzZjdmN2QyOTU0ZWZhYTlmNTg3Nzc0ODdkN2QxOWY5ZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNjIyODJmZDM2NDJiNDMzMzY2NTYyYmE1OWE1M2JhNWI5MzAyNTdlNzNlM2FkNThiOTZkNjhjMjcwMGYzOTc1OCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI3MDVmNTZjMGQ3MmYzNWJlYTlkMjEzMGU1NjJlNTBmOTlhMTNiOTExNmQ5ZjlkNGQzZDdlNWY5N2RhYzlkN2Q4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiN2YyMmZiNDM3ZTBjZjAwZDFjMGNlZWU2MmEwYmI3ZWYwYWYzNTQ4ZDM0MjI5ZWZmZDFjMDU1Y2I5ZTJhZDI3YSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcG9wdWxhdGlvbnMubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImM0NDk2MGNmMjI0YjY0ODNiMzQwMTU5YWNjM2YwMWUxYzdkNzM4ZjRiNjEwMTgyZDMwOWM2Zjc0ZjRmMWJhYjEiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIKICAgICAgXQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCHGw/myeqEVuOCOjZhu4CyeMGEJMrcBlIUEKygqBOlqs7lkEazg/emr5ubSBK1QtECMAkrD0w3DeZkq+X+ivOOHETQE7cOjbvvITzDbbGV5UuAjs/B8YhFI/Wq0/3lSVE1eQ==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1jdXN0b20tcG9wdWxhdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMjJiMTNjMGUzN2MxYWYxZDM1MzQ4MTVmOGI3OGMxOGIxN2M4NmIxZjZjODdhYzE2ZGNiYmJkNDAwNDAwOTJlMCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjAyMmViYTQ4NGFhNTMzNWFhYzE4ODc3OTk2MWZlMDhjMTVhNTU5NWYyNmI2NzgxMzEyNDNlNDFkNzk3M2I4MTIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjIyODJmZDM2NDJiNDMzMzY2NTYyYmE1OWE1M2JhNWI5MzAyNTdlNzNlM2FkNThiOTZkNjhjMjcwMGYzOTc1OCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjcwNWY1NmMwZDcyZjM1YmVhOWQyMTMwZTU2MmU1MGY5OWExM2I5MTE2ZDlmOWQ0ZDNkN2U1Zjk3ZGFjOWQ3ZDgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BvcHVsYXRpb25zLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3ZjIyZmI0MzdlMGNmMDBkMWMwY2VlZTYyYTBiYjdlZjBhZjM1NDhkMzQyMjllZmZkMWMwNTVjYjllMmFkMjdhIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNTIzMDU4NTJhOTE1ODk5NDI0NTUxYTY4ZDAyODQ4ZTU2ZGYxMzZkM2QyMjM0ZjNhNDE3YzFmZDJhYTNmOGJjZSIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQD/TX010LpYPu2x2UliUA2YcUsLXc3lO/dU3RJPLzoxFQQrKc1BNJa3ARUViOe6/OECMB6g8LSywJK3JynkVBFiQe8PKd4ZNLhcmOzIilV/s37ulSf3XKOp1NUnEE/1Gmx10g==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-data-science/BENCHMARK.md b/skills/kumo-data-science/BENCHMARK.md index e50d93787..b60d051e5 100644 --- a/skills/kumo-data-science/BENCHMARK.md +++ b/skills/kumo-data-science/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 91.7% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 87.5% → 100.0% (+12.5 points) | -| Correctness | 52.5% → 100.0% (+47.5 points) | 47.5% → 92.5% (+45.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 42.6% → 100.0% (+57.4 points) | 38.8% → 80.4% (+41.6 points) | -| Efficiency | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 98.4% — baseline ran, but no comparable score was available; uplift unavailable | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 57.5% → 100.0% (+42.5 points) | 50.0% → 97.5% (+47.5 points) | +| Discoverability | 99.3% — baseline ran, but no comparable score was available; uplift unavailable | 92.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 41.4% → 97.9% (+56.5 points) | 36.2% → 93.8% (+57.6 points) | +| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.0% → 92.4% (-4.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.0% → 92.6% (-4.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 83.2% → 90.3% (+7.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 80.4% → 90.3% (+9.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,25 +80,25 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,593,311 | 932,554 | +660,757 | +70.85% | skill 8/8; base 8/8 | -| claude-code | kds-01 | 217,339 | 143,640 | +73,699 | +51.31% | skill 1/1; base 1/1 | -| claude-code | kds-02 | 184,163 | 166,351 | +17,812 | +10.71% | skill 1/1; base 1/1 | -| claude-code | kds-03 | 237,291 | 193,342 | +43,949 | +22.73% | skill 1/1; base 1/1 | -| claude-code | kds-04 | 316,252 | 108,091 | +208,161 | +192.58% | skill 1/1; base 1/1 | -| claude-code | kds-05 | 216,613 | 70,836 | +145,777 | +205.80% | skill 1/1; base 1/1 | -| claude-code | kds-06 | 216,619 | 107,275 | +109,344 | +101.93% | skill 1/1; base 1/1 | -| claude-code | kds-07 | 169,166 | 107,391 | +61,775 | +57.52% | skill 1/1; base 1/1 | -| claude-code | kds-08 | 35,868 | 35,628 | +240 | +0.67% | skill 1/1; base 1/1 | -| codex | All cases | 603,483 | 3,473,502 | -2,870,019 | -82.63% | skill 8/8; base 8/8 | -| codex | kds-01 | 71,810 | 45,041 | +26,769 | +59.43% | skill 1/1; base 1/1 | -| codex | kds-02 | 72,570 | 2,079,924 | -2,007,354 | -96.51% | skill 1/1; base 1/1 | -| codex | kds-03 | 119,713 | 1,151,989 | -1,032,276 | -89.61% | skill 1/1; base 1/1 | -| codex | kds-04 | 71,905 | 41,272 | +30,633 | +74.22% | skill 1/1; base 1/1 | -| codex | kds-05 | 86,631 | 26,266 | +60,365 | +229.82% | skill 1/1; base 1/1 | -| codex | kds-06 | 48,715 | 40,986 | +7,729 | +18.86% | skill 1/1; base 1/1 | -| codex | kds-07 | 118,454 | 74,560 | +43,894 | +58.87% | skill 1/1; base 1/1 | -| codex | kds-08 | 13,685 | 13,464 | +221 | +1.64% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 2,196,794 | 4,406,056 | -2,209,262 | -50.14% | skill 16/16; base 16/16 | +| claude-code | All cases | 1,456,211 | 933,207 | +523,004 | +56.04% | skill 8/8; base 8/8 | +| claude-code | kds-01 | 225,387 | 107,911 | +117,476 | +108.86% | skill 1/1; base 1/1 | +| claude-code | kds-02 | 183,714 | 201,375 | -17,661 | -8.77% | skill 1/1; base 1/1 | +| claude-code | kds-03 | 229,800 | 158,004 | +71,796 | +45.44% | skill 1/1; base 1/1 | +| claude-code | kds-04 | 222,519 | 108,540 | +113,979 | +105.01% | skill 1/1; base 1/1 | +| claude-code | kds-05 | 217,354 | 107,133 | +110,221 | +102.88% | skill 1/1; base 1/1 | +| claude-code | kds-06 | 169,879 | 107,398 | +62,481 | +58.18% | skill 1/1; base 1/1 | +| claude-code | kds-07 | 171,634 | 107,252 | +64,382 | +60.03% | skill 1/1; base 1/1 | +| claude-code | kds-08 | 35,924 | 35,594 | +330 | +0.93% | skill 1/1; base 1/1 | +| codex | All cases | 601,311 | 2,529,905 | -1,928,594 | -76.23% | skill 8/8; base 8/8 | +| codex | kds-01 | 70,062 | 45,543 | +24,519 | +53.84% | skill 1/1; base 1/1 | +| codex | kds-02 | 96,604 | 1,481,282 | -1,384,678 | -93.48% | skill 1/1; base 1/1 | +| codex | kds-03 | 122,113 | 819,648 | -697,535 | -85.10% | skill 1/1; base 1/1 | +| codex | kds-04 | 94,265 | 41,384 | +52,881 | +127.78% | skill 1/1; base 1/1 | +| codex | kds-05 | 71,914 | 30,254 | +41,660 | +137.70% | skill 1/1; base 1/1 | +| codex | kds-06 | 83,337 | 41,036 | +42,301 | +103.08% | skill 1/1; base 1/1 | +| codex | kds-07 | 49,343 | 57,280 | -7,937 | -13.86% | skill 1/1; base 1/1 | +| codex | kds-08 | 13,673 | 13,478 | +195 | +1.45% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,057,522 | 3,463,112 | -1,405,590 | -40.59% | skill 16/16; base 16/16 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -121,14 +121,14 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - **HIGH** DUPLICATE/duplicate: Duplicate content found across validate/atomic_question_check.md and validate/examples/bad_request.md: "### Fail 1 — Multiple targets bundled" in validate/atomic_question_check.md (lines 42-50) vs "## 6. Bundled goals → REJECT" in validate/examples/bad_request.md (lines 64-68) (`validate/atomic_question_check.md:42`) -- **HIGH** DUPLICATE/duplicate: Duplicate content found across accept/intake_contract.md and validate/examples/good_request.md: - "## Canonical good example (`train`)" in accept/intake_contract.md (lines 137-148) - vs "## Input" in validate/examples/good_request.md (lines 3-12) - vs "## Node 1a — Required fields: PASS" in validate/examples/good_request.md (lines 13-19) (`accept/intake_contract.md:137`) - **HIGH** DUPLICATE/duplicate: Duplicate content found within accept/batch_inference.md: "## Node 1a — fields" in accept/batch_inference.md (lines 7-29) vs "### `batch_inference` — required fields (ask if missing)" in accept/batch_inference.md (lines 30-42) vs "### `batch_inference` — resolved, defaulted, or inferred (never ask)" in accept/batch_inference.md (lines 43-54) (`accept/batch_inference.md:7`) +- **HIGH** DUPLICATE/duplicate: Duplicate content found across accept/intake_contract.md and validate/examples/good_request.md: + "## Canonical good example (`train`)" in accept/intake_contract.md (lines 137-148) + vs "## Input" in validate/examples/good_request.md (lines 3-12) + vs "## Node 1a — Required fields: PASS" in validate/examples/good_request.md (lines 13-19) (`accept/intake_contract.md:137`) - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-data-science/SKILL.md`) - 25 additional finding(s) are available in the full evaluation artifacts. diff --git a/skills/kumo-data-science/skill-card.md b/skills/kumo-data-science/skill-card.md index eef55ab15..20b3d149a 100644 --- a/skills/kumo-data-science/skill-card.md +++ b/skills/kumo-data-science/skill-card.md @@ -41,7 +41,7 @@ Mitigation: Review and scan skill before deployment.
**Output Type(s):** [Analysis, Code, Files]
**Output Format:** [Markdown replies (graph and task proposals, ask-backs) and one generated Jupyter notebook or Python script using the kumo.engine API]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Generates code only until the user confirms the graph and task; runs the artifact only on explicit request]
+**Other Properties Related to Output:** [Generates code only after the user confirms the graph and task; runs the artifact only on explicit request]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -76,23 +76,29 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 91.7% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 87.5% → 100.0% (+12.5 points) | -| Correctness | 52.5% → 100.0% (+47.5 points) | 47.5% → 92.5% (+45.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 42.6% → 100.0% (+57.4 points) | 38.8% → 80.4% (+41.6 points) | -| Efficiency | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 98.4% — baseline ran, but no comparable score was available; uplift unavailable | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 57.5% → 100.0% (+42.5 points) | 50.0% → 97.5% (+47.5 points) | +| Discoverability | 99.3% — baseline ran, but no comparable score was available; uplift unavailable | 92.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 41.4% → 97.9% (+56.5 points) | 36.2% → 93.8% (+57.6 points) | +| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | -Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; uplift is derived from the displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. #### Comparison details +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.0% → 92.4% (-4.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.0% → 92.6% (-4.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 83.2% → 90.3% (+7.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 80.4% → 90.3% (+9.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -107,7 +113,7 @@ Baseline is the same task attempted without the target skill. Scores are rounded **Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. ## Skill Version(s):
-0.1.0 (source: pyproject.toml); skill build marker 2026-11-17-5026ce47-dirty (source: SKILL.md); git 7cbda7af (committed 2026-11-18)
+0.1.0 (source: pyproject.toml); skill build marker 2026-11-18-83ce9a89 (source: SKILL.md); git 8aaa6cb3 (committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-data-science/skill.oms.sig b/skills/kumo-data-science/skill.oms.sig index 85af9b7a4..192dd7d34 100644 --- a/skills/kumo-data-science/skill.oms.sig +++ b/skills/kumo-data-science/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1kYXRhLXNjaWVuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNzU2OWI2ZjIyMmRhNjA5NjA0NDU5OTIxZjAxNTk4MWMwNWI4YjQ1OWUyZWYzNjk5YjY4MWFkYjY4ZjE3YWVhNCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzMzExMTdkMzRjNDllNmNhY2JkNmE5ZmU0MjhiZjQzM2RlODNlY2I1MmRkMzJlNTEwYmU4Y2VhZjljZTYwZDI1IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlNGJiOWMzODljNmM3MGVjMGQ5MDkxNmExOGU5NDJlZTM5Y2I5ZmE3N2ExNjQ0NzlmMDcwOTk0ZTdkZDBlYzI3IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjJkNzVhYmEwZmE0ZmUxOWI4ODU2NTMyODBlMWE1MmQxM2IwN2EwZTEzM2RjNDM0ZDFiMmJkN2FiMGY5ZjZiYjgiLAogICAgICAgICJuYW1lIjogImFjY2VwdC9iYXRjaF9pbmZlcmVuY2UubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmZWE3YmJkODY1YjgzNjY0MTEwN2Y3NDAzNDgyNmIxZjU0YTE0ZDE0ZjM4YjE3ZWFjMGZkZGE4MDg5NDEyMmQxIiwKICAgICAgICAibmFtZSI6ICJhY2NlcHQvaW50YWtlX2NvbnRyYWN0Lm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDNiNGM5MDc0MTVlYTQzNjBlMDlhNGFhMjJlOWNhOGU2OGJkMjNmOGI0YzQ3Y2NmOWQxYjk1ZDhhYTYxYmU5ZiIsCiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L3ByZWZsaWdodC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImY5ZjJkZjljYWE0ZDU2OWU0M2U0ZDM0ZjcwOTliMjE1MTFmYWMxYWIzOWVkNjIyNzMyNDhmYjYxMWI5Y2JkM2UiLAogICAgICAgICJuYW1lIjogImFjY2VwdC9zZW1hbnRpY192aWV3Lm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjY0MDRkOWNiMTNkMDA5M2I2NzAxZGZkNGIxOTMzNzEwM2Q4ZDNiODYwNzVlMTExMzkzNmQzYjQ0ZTk5Y2NhOSIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImM2YTM2ODQ1NGZhOWFkZWViYTBiZGRhOThiZDNkMjE0ZjJiOTJhNWYxMjgzNTI3NDVjM2QyMmViOTA3YjViNmIiLAogICAgICAgICJuYW1lIjogImhvc3RzL2dlbmVyaWMvYWRhcHRlci5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjBjZDA0NzkzNDNiNmM4MWE1YWU5ZjI4Zjk5N2ZlMGZhMDYxZGFjODA1ZDI0ODlmZGUxMTg2NzMxZjBlMDE2ZGYiLAogICAgICAgICJuYW1lIjogImhvc3RzL2dlbmVyaWMvcnVudGltZS5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImM1ZWRlYTBlYzhiNzk1MmI4ZmU1ZTViYTZjMDcwMjUwNzgwMjI5MTBjN2U1MDU4ZmMwMDAwYWE2NDY3MGY1M2UiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2NTA2ZTY2MWJjMTFjMjVlYjY2YTdiN2MzNWEzNDAzMTM5NTdkMzI0YmEzODY3ZTk1MDg2YjExNGU5MmQyMDRhIiwKICAgICAgICAibmFtZSI6ICJ2YWxpZGF0ZS9hdG9taWNfcXVlc3Rpb25fY2hlY2subWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3YmQzOTM0ZTFiOWMyMmE0YzM4YjM3YmNiOGM1OTcyOTJkZGVlODIyMDYxZDlmZWVlZTY1N2JkOTA1MDM1Y2Q0IiwKICAgICAgICAibmFtZSI6ICJ2YWxpZGF0ZS9leGFtcGxlcy9iYWRfcmVxdWVzdC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjg5MmQwZmJlYjQyYTgwMmQzMzZmZmIyMzA5ZDVlOTk5YjM4Y2ZiNDg3ZDM2OWMzODkyOTQ5ZjNjYzFiZWQ4ZTYiLAogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2V4YW1wbGVzL2dvb2RfcmVxdWVzdC5tZCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMCnuu90cyjzIwuc9siUNkR3LgxyEXxXp7ZBbsYxVpxOTQJbQY5DLH+3SZuhCFQo3UAIxAK05uplVBCVMowcOKH7kA6x6kDvgAsK/ju6HXHrccHIqkTfln11i2wRU6mcRQnPu0w==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1kYXRhLXNjaWVuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiOWNiYjliOTAwNDRlM2EyYzQ5ZTc3ZTJjNDUyOWYxZGY3YzI1YTcxMmY3ZmI4OGFhMzdmN2NmZTc1MDhiMjI5ZSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZTA4ZDI3MjJkZTExZDc1NzVlYzI4ODNmMzBhNmZhNjAxNGZlZTJmMmFkODE5MTNiOTRmMjdiZjIyMWZmODRkYSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3ZDUwZTA0YWE4YTU1ODFiMjVhNDY4YTYxZmQ5YjM1MTJkNDM4MDRiZjc0ZDczNWYxMjA0NzRkMzhkYWM3Y2U2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L2JhdGNoX2luZmVyZW5jZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMmQ3NWFiYTBmYTRmZTE5Yjg4NTY1MzI4MGUxYTUyZDEzYjA3YTBlMTMzZGM0MzRkMWIyYmQ3YWIwZjlmNmJiOCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImFjY2VwdC9pbnRha2VfY29udHJhY3QubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImZlYTdiYmQ4NjViODM2NjQxMTA3Zjc0MDM0ODI2YjFmNTRhMTRkMTRmMzhiMTdlYWMwZmRkYTgwODk0MTIyZDEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJhY2NlcHQvcHJlZmxpZ2h0Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkM2I0YzkwNzQxNWVhNDM2MGUwOWE0YWEyMmU5Y2E4ZTY4YmQyM2Y4YjRjNDdjY2Y5ZDFiOTVkOGFhNjFiZTlmIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L3NlbWFudGljX3ZpZXcubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImY5ZjJkZjljYWE0ZDU2OWU0M2U0ZDM0ZjcwOTliMjE1MTFmYWMxYWIzOWVkNjIyNzMyNDhmYjYxMWI5Y2JkM2UiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2NjQwNGQ5Y2IxM2QwMDkzYjY3MDFkZmQ0YjE5MzM3MTAzZDhkM2I4NjA3NWUxMTEzOTM2ZDNiNDRlOTljY2E5IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiaG9zdHMvZ2VuZXJpYy9hZGFwdGVyLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJjNmEzNjg0NTRmYTlhZGVlYmEwYmRkYTk4YmQzZDIxNGYyYjkyYTVmMTI4MzUyNzQ1YzNkMjJlYjkwN2I1YjZiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiaG9zdHMvZ2VuZXJpYy9ydW50aW1lLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlMjM5ZTJkODQ2NWFiYWI3OWMyN2YyNmIyMWViY2M3Nzc0MzM5YjQxMDNhMDM2MGY4MjJmNzhhZmMyNGNlOTBiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzE5NTRiMzFiODAzNjg4ZmViMTUxNTZiNzBmMGQxYmQzNGQ5ZmZjYzRlZTIyMmM3OGVjMWM2Y2Y5MjE1NWRiMiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2F0b21pY19xdWVzdGlvbl9jaGVjay5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjUwNmU2NjFiYzExYzI1ZWI2NmE3YjdjMzVhMzQwMzEzOTU3ZDMyNGJhMzg2N2U5NTA4NmIxMTRlOTJkMjA0YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2V4YW1wbGVzL2JhZF9yZXF1ZXN0Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3YmQzOTM0ZTFiOWMyMmE0YzM4YjM3YmNiOGM1OTcyOTJkZGVlODIyMDYxZDlmZWVlZTY1N2JkOTA1MDM1Y2Q0IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAidmFsaWRhdGUvZXhhbXBsZXMvZ29vZF9yZXF1ZXN0Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI4OTJkMGZiZWI0MmE4MDJkMzM2ZmZiMjMwOWQ1ZTk5OWIzOGNmYjQ4N2QzNjljMzg5Mjk0OWYzY2MxYmVkOGU2IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDZezCWX0j2hma/i4Yfp6dahzVDS7Z5ogIIv9lI0ulzp75pPNQsf2u+5JQMRtv7JEcCMFFm5UW8A9K1p+7TMzF//1Jbutj7HFhyg271tZNG7V08gs89zX9e+33LiXPr3DbltQ==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-graph-visualization/BENCHMARK.md b/skills/kumo-graph-visualization/BENCHMARK.md index c78575965..d58f8c9a7 100644 --- a/skills/kumo-graph-visualization/BENCHMARK.md +++ b/skills/kumo-graph-visualization/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 90.7% — baseline ran, but no comparable score was available; uplift unavailable | 79.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 88.2% — baseline ran, but no comparable score was available; uplift unavailable | 79.7% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 32.0% → 100.0% (+68.0 points) | 8.0% → 72.0% (+64.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 91.3% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 30.8% → 60.0% (+29.2 points) | 13.3% → 37.5% (+24.2 points) | -| Efficiency | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | 96.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 72.0% (+44.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 30.8% → 60.0% (+29.2 points) | 30.0% → 37.5% (+7.5 points) | +| Efficiency | 81.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.5% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 92.6% → 97.5% (+4.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 97.6% (-0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 93.3% → 92.9% (-0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 91.6% → 93.0% (+1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 808,922 | 788,473 | +20,449 | +2.59% | skill 5/5; base 5/5 | -| claude-code | kviz-01 | 145,987 | 177,996 | -32,009 | -17.98% | skill 1/1; base 1/1 | -| claude-code | kviz-02 | 147,637 | 107,276 | +40,361 | +37.62% | skill 1/1; base 1/1 | -| claude-code | kviz-03 | 147,145 | 182,178 | -35,033 | -19.23% | skill 1/1; base 1/1 | -| claude-code | kviz-04 | 109,826 | 107,310 | +2,516 | +2.34% | skill 1/1; base 1/1 | -| claude-code | kviz-05 | 258,327 | 213,713 | +44,614 | +20.88% | skill 1/1; base 1/1 | -| codex | All cases | 272,278 | 205,989 | +66,289 | +32.18% | skill 5/5; base 5/5 | -| codex | kviz-01 | 57,309 | 40,444 | +16,865 | +41.70% | skill 1/1; base 1/1 | -| codex | kviz-02 | 42,456 | 57,036 | -14,580 | -25.56% | skill 1/1; base 1/1 | -| codex | kviz-03 | 73,002 | 27,052 | +45,950 | +169.86% | skill 1/1; base 1/1 | -| codex | kviz-04 | 57,045 | 41,055 | +15,990 | +38.95% | skill 1/1; base 1/1 | -| codex | kviz-05 | 42,466 | 40,402 | +2,064 | +5.11% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,081,200 | 994,462 | +86,738 | +8.72% | skill 10/10; base 10/10 | +| claude-code | All cases | 842,402 | 751,532 | +90,870 | +12.09% | skill 5/5; base 5/5 | +| claude-code | kviz-01 | 145,858 | 106,564 | +39,294 | +36.87% | skill 1/1; base 1/1 | +| claude-code | kviz-02 | 148,209 | 107,632 | +40,577 | +37.70% | skill 1/1; base 1/1 | +| claude-code | kviz-03 | 185,661 | 180,056 | +5,605 | +3.11% | skill 1/1; base 1/1 | +| claude-code | kviz-04 | 109,587 | 143,724 | -34,137 | -23.75% | skill 1/1; base 1/1 | +| claude-code | kviz-05 | 253,087 | 213,556 | +39,531 | +18.51% | skill 1/1; base 1/1 | +| codex | All cases | 271,299 | 225,930 | +45,369 | +20.08% | skill 5/5; base 5/5 | +| codex | kviz-01 | 57,292 | 54,986 | +2,306 | +4.19% | skill 1/1; base 1/1 | +| codex | kviz-02 | 27,459 | 46,274 | -18,815 | -40.66% | skill 1/1; base 1/1 | +| codex | kviz-03 | 72,377 | 26,738 | +45,639 | +170.69% | skill 1/1; base 1/1 | +| codex | kviz-04 | 56,957 | 57,507 | -550 | -0.96% | skill 1/1; base 1/1 | +| codex | kviz-05 | 57,214 | 40,425 | +16,789 | +41.53% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,113,701 | 977,462 | +136,239 | +13.94% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-graph-visualization/skill-card.md b/skills/kumo-graph-visualization/skill-card.md index 5c5d231c7..a597601a9 100644 --- a/skills/kumo-graph-visualization/skill-card.md +++ b/skills/kumo-graph-visualization/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill with coding agents to view a Kumo graph's tables and relationships in a notebook, export it as SVG, PDF or Markdown, and read its schema while deciding on graph changes.
+Developers and data scientists using Kumo notebooks or SDK scripts use this skill to display or export graph diagrams of tables and relationships (SVG, PDF, or Markdown/Mermaid) and to inspect graph schemas while iterating on models.
### Deployment Geography for Use:
Global
@@ -25,15 +25,16 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+- [Graph construction (kumoai-reference)](../kumoai-reference/references/graph_construction.md)
+- [Kumo host runtimes (kumo-data-science)](../kumo-data-science/hosts/)
- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
## Skill Output:
-**Output Type(s):** [Code, Analysis]
-**Output Format:** [Markdown with inline Python code blocks; SVG, PDF or Markdown graph files when exported]
+**Output Type(s):** [Code, Files]
+**Output Format:** [Markdown with inline Python code blocks; SVG, PDF, or Markdown graph exports]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Source sample values are off unless the user asks for them.]
+**Other Properties Related to Output:** [Sample values are omitted unless the user requests them]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -42,7 +43,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`), each compared against a no-skill baseline. Evaluator version `1.6.10`, evaluation date 2026-11-18.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod (`k8s-sandbox`); evaluator version `1.6.10`, evaluated 2026-11-18. Tier 2 and Tier 3 evidence required for publication.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -53,33 +54,33 @@ Reported benchmark dimensions:
- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Final-answer correctness against the reference answer.
-- `goal_accuracy`: Whether the user's goal was achieved.
-- `behavior_check`: Whether the expected workflow behavior was followed.
-- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
+**Overall verdict: PASS — Recommended for publication** + +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. + ### Results at a Glance | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 90.7% — baseline ran, but no comparable score was available; uplift unavailable | 79.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 88.2% — baseline ran, but no comparable score was available; uplift unavailable | 79.7% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 32.0% → 100.0% (+68.0 points) | 8.0% → 72.0% (+64.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 91.3% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 30.8% → 60.0% (+29.2 points) | 13.3% → 37.5% (+24.2 points) | -| Efficiency | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | 96.4% — baseline ran, but no comparable score was available; uplift unavailable | - -**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. +| Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 72.0% (+44.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 30.8% → 60.0% (+29.2 points) | 30.0% → 37.5% (+7.5 points) | +| Efficiency | 81.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.5% — baseline ran, but no comparable score was available; uplift unavailable | -Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. - -A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. #### Comparison details @@ -88,9 +89,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 92.6% → 97.5% (+4.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 97.6% (-0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 93.3% → 92.9% (-0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 91.6% → 93.0% (+1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -100,11 +101,8 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | -**Overall verdict:** PASS — Recommended for publication
-**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report.
- ## Skill Version(s):
-0.1.0 (source: pyproject.toml; git SHA 7cbda7af, committed 2026-11-18)
+0.1.0 (source: pyproject.toml; git SHA 8aaa6cb3, committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-graph-visualization/skill.oms.sig b/skills/kumo-graph-visualization/skill.oms.sig index cbd140d0e..72ee93c08 100644 --- a/skills/kumo-graph-visualization/skill.oms.sig +++ b/skills/kumo-graph-visualization/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1ncmFwaC12aXN1YWxpemF0aW9uIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImJkYjViYTFmYWUxMzA0YTY0OTFmYjlhYzEwYjRiNTkyNzliMjVlYThmYTY2ZjlhYTM5NDU0ZTA2NDE2ODUxNTEiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImNjYjYwNDM3YTFlMGMzNWM2ZDc4NjNlMDlkOTBmNGNmN2U1OWY4ODUxZGM5ODFjYzEzYWRjYTM0ZmQ3M2IzZmEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOTU2M2MyZDVkNGQzMmJhYjAyOGE0Zjk2YmExNWU0NDVkN2FlYzIyZjEzZjRkYmY0NDc2NWVjZjUzYjBjOGZmNyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjc3NGEyNzhhNjY5MGZlMGJmNzEwMDFkNzYwOTVkMzY2ZDEwYTgwM2M4YTY2ODY1ZGRmMzRhMzJlNzVlNmI5OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkMzFiOTk3MGUxNmY1ZGRjMDRjMDU1NmVkMzUzOTgzYjIyMzI3MjhkNmI0ZmQ4MGMyZmQ5ZjAyMzlmZmJiMGY4IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIKICAgICAgXSwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDBw6wiYYjBbzGmdGkCOSuNrRTlN7rH01TPGF7qphMNKNEfoQt7uBN35qhK5Sn0oWcCMQDVywIMWOB51cgKBowea1PIcbRXCeGH5Xq995h6v6k27Y/sxt2HFtOmuj9rUqGoejY=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1ncmFwaC12aXN1YWxpemF0aW9uIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImZjYzkxMGUxY2Q1YmUwY2FhZmI2Y2M2NDA0ZDIwNmNhZDQwZDk3M2M0ODk4NmU1N2E5NzA3NzgyZDA3MjE1NjgiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2RhYmVkZjJlMWY2MTMxMjY1ODZlYjdiNGEwOTQzODRjNTEyZjQxNmI0N2M4OWY0MWQxOTc1NjcxYTIwMjYyMSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI5NTYzYzJkNWQ0ZDMyYmFiMDI4YTRmOTZiYTE1ZTQ0NWQ3YWVjMjJmMTNmNGRiZjQ0NzY1ZWNmNTNiMGM4ZmY3IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiNzc0YTI3OGE2NjkwZmUwYmY3MTAwMWQ3NjA5NWQzNjZkMTBhODAzYzhhNjY4NjVkZGYzNGEzMmU3NWU2Yjk5YiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjM5Y2Q3Y2VhMjUzNzUxYjg3MmQ0ZDlhMDJmY2QyM2MwNjNhZDg3YjAzMGJmZGNlNjE1MWM4YTNjMDZiZmFiZjciLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGh1YiIKICAgICAgXQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQD36oDp24dEG6QXJda30i8md8JdxSq1VjKhg0VXvblj/fM1BkmIyyILc4vowoXXCywCMQCyBcSTEbEwA+8CjfHO0f+AA9HYaGv9lJGrRQWNI2ASDWn+Alz1VX9ZLBhZGh+/jso=","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-explanations/BENCHMARK.md b/skills/kumo-model-explanations/BENCHMARK.md index e53d9c40b..47f847e03 100644 --- a/skills/kumo-model-explanations/BENCHMARK.md +++ b/skills/kumo-model-explanations/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | 88.8% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 43.3% → 100.0% (+56.7 points) | 56.7% → 93.3% (+36.6 points) | +| Correctness | 40.0% → 100.0% (+60.0 points) | 56.7% → 93.3% (+36.6 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 25.0% → 60.4% (+35.4 points) | 32.0% → 68.1% (+36.1 points) | -| Efficiency | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 27.8% → 64.6% (+36.8 points) | 50.0% → 59.7% (+9.7 points) | +| Efficiency | 97.8% — baseline ran, but no comparable score was available; uplift unavailable | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 95.9% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 95.6% (-2.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.3% → 91.9% (+2.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.8% → 91.9% (+2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,21 +80,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 891,246 | 716,277 | +174,969 | +24.43% | skill 6/6; base 6/6 | -| claude-code | kexpl-01 | 158,307 | 70,650 | +87,657 | +124.07% | skill 1/1; base 1/1 | -| claude-code | kexpl-02 | 159,005 | 108,056 | +50,949 | +47.15% | skill 1/1; base 1/1 | -| claude-code | kexpl-03 | 158,205 | 107,250 | +50,955 | +47.51% | skill 1/1; base 1/1 | -| claude-code | kexpl-04 | 160,396 | 144,461 | +15,935 | +11.03% | skill 1/1; base 1/1 | -| claude-code | kexpl-05 | 148,103 | 178,444 | -30,341 | -17.00% | skill 1/1; base 1/1 | -| claude-code | kexpl-06 | 107,230 | 107,416 | -186 | -0.17% | skill 1/1; base 1/1 | -| codex | All cases | 412,061 | 335,024 | +77,037 | +22.99% | skill 6/6; base 6/6 | -| codex | kexpl-01 | 80,627 | 70,726 | +9,901 | +14.00% | skill 1/1; base 1/1 | -| codex | kexpl-02 | 82,514 | 41,509 | +41,005 | +98.79% | skill 1/1; base 1/1 | -| codex | kexpl-03 | 80,965 | 60,631 | +20,334 | +33.54% | skill 1/1; base 1/1 | -| codex | kexpl-04 | 81,379 | 64,915 | +16,464 | +25.36% | skill 1/1; base 1/1 | -| codex | kexpl-05 | 45,308 | 56,654 | -11,346 | -20.03% | skill 1/1; base 1/1 | -| codex | kexpl-06 | 41,268 | 40,589 | +679 | +1.67% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,303,307 | 1,051,301 | +252,006 | +23.97% | skill 12/12; base 12/12 | +| claude-code | All cases | 966,880 | 823,367 | +143,513 | +17.43% | skill 6/6; base 6/6 | +| claude-code | kexpl-01 | 159,703 | 142,288 | +17,415 | +12.24% | skill 1/1; base 1/1 | +| claude-code | kexpl-02 | 158,941 | 107,859 | +51,082 | +47.36% | skill 1/1; base 1/1 | +| claude-code | kexpl-03 | 116,594 | 142,990 | -26,396 | -18.46% | skill 1/1; base 1/1 | +| claude-code | kexpl-04 | 160,267 | 144,212 | +16,055 | +11.13% | skill 1/1; base 1/1 | +| claude-code | kexpl-05 | 226,120 | 178,315 | +47,805 | +26.81% | skill 1/1; base 1/1 | +| claude-code | kexpl-06 | 145,255 | 107,703 | +37,552 | +34.87% | skill 1/1; base 1/1 | +| codex | All cases | 411,790 | 298,681 | +113,109 | +37.87% | skill 6/6; base 6/6 | +| codex | kexpl-01 | 81,391 | 66,405 | +14,986 | +22.57% | skill 1/1; base 1/1 | +| codex | kexpl-02 | 81,040 | 41,166 | +39,874 | +96.86% | skill 1/1; base 1/1 | +| codex | kexpl-03 | 81,498 | 53,873 | +27,625 | +51.28% | skill 1/1; base 1/1 | +| codex | kexpl-04 | 81,678 | 54,065 | +27,613 | +51.07% | skill 1/1; base 1/1 | +| codex | kexpl-05 | 45,305 | 42,610 | +2,695 | +6.32% | skill 1/1; base 1/1 | +| codex | kexpl-06 | 40,878 | 40,562 | +316 | +0.78% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,378,670 | 1,122,048 | +256,622 | +22.87% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-explanations/skill-card.md b/skills/kumo-model-explanations/skill-card.md index b8516d5ca..45674d2c4 100644 --- a/skills/kumo-model-explanations/skill-card.md +++ b/skills/kumo-model-explanations/skill-card.md @@ -72,14 +72,18 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | 88.8% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 43.3% → 100.0% (+56.7 points) | 56.7% → 93.3% (+36.6 points) | +| Correctness | 40.0% → 100.0% (+60.0 points) | 56.7% → 93.3% (+36.6 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 25.0% → 60.4% (+35.4 points) | 32.0% → 68.1% (+36.1 points) | -| Efficiency | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 27.8% → 64.6% (+36.8 points) | 50.0% → 59.7% (+9.7 points) | +| Efficiency | 97.8% — baseline ran, but no comparable score was available; uplift unavailable | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | -Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. ### Comparison details @@ -88,12 +92,12 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 95.9% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 95.6% (-2.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.3% → 91.9% (+2.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.8% → 91.9% (+2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
-0.1.0 (source: pyproject.toml; git SHA 7cbda7af, committed 2026-11-18)
+0.1.0 (source: pyproject.toml; git SHA 8aaa6cb3, committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-model-explanations/skill.oms.sig b/skills/kumo-model-explanations/skill.oms.sig index a07d73fd4..f97db8fa3 100644 --- a/skills/kumo-model-explanations/skill.oms.sig +++ b/skills/kumo-model-explanations/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1leHBsYW5hdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNWIwNjUxZmFjNzliZjUxMWUxZjIyMzQyNmJhZmQwNzYwMzcwNzNhNzFhMGRmZWE3NDRiYzNjZjc2ZWM3ZGUxMyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjUzNjRmM2MzYzZkYTY3ZjUzM2UzMDRkNDUwZDgyODFmNGVkZjFkMTNlNDRiZmJhMGQ2ZDUxZjExYTkzZTc5YmUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiY2I3NGI3ZGRjZGQzYWUwNmM1MmJjODUyY2Q5Yzk3OWU5NGJhNjQwYzZlY2U4MjJmNWY0OTJiYmZmOTdkYWEwYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjk2NzRhNzFhMWExYmYwNmU1ZjNhMDdlNGNjZWRlNTdkMTMzNjYzOWY0NGMyMWYwMzQzYWQ3MmE3ZTA3MmRiMTciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9hcnRpZmFjdHMubWQiLAogICAgICAgICJkaWdlc3QiOiAiNTBjYjY4ZmFiZDc0YzhlMWU5MmM0YzgzMGVlNTA5ZWRjYjY2NTRiNzBmOThjMjdjZThmZTM0Yzg5OGVhNzE3YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjkzM2M5ZDk5YzRkMGU3NDFkNGI3ZDA3ZjVjYzc3OTI3OGMxZTQ5MDIxNDEyMzA0OTlhMjY5MDlmNGNjY2JmOGQiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQC1NAaO59p2CHGCR4qHXfcn/Qc6MSgWSW+TFtkftb0UdoBbk4t4JPpmiZb7zHXecR4CMQDhAtGyvvTpYmaHMa32vrH5Cp3uWtX/49EAWaf8gFFXTrshgyhs6AoIkUy2WVWHPl8=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1leHBsYW5hdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMjU4YTEzMjkyZmM1MTQ1YjU5ZTRjOTE0NjMxMTUzYmUwMDViZDNlYzNmYmU2OTllY2E0MGU1MDgxMWI4OWI1MyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogImJmOTk0YWM0ODkxYzc0MzI3YmIzNGE5NTBlYjZlYjcwNmJkNDlkNjkwNWQyNzMxMGEyMzYzODliMjAyMjg1NTgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiY2I3NGI3ZGRjZGQzYWUwNmM1MmJjODUyY2Q5Yzk3OWU5NGJhNjQwYzZlY2U4MjJmNWY0OTJiYmZmOTdkYWEwYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjk2NzRhNzFhMWExYmYwNmU1ZjNhMDdlNGNjZWRlNTdkMTMzNjYzOWY0NGMyMWYwMzQzYWQ3MmE3ZTA3MmRiMTciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9hcnRpZmFjdHMubWQiLAogICAgICAgICJkaWdlc3QiOiAiNTBjYjY4ZmFiZDc0YzhlMWU5MmM0YzgzMGVlNTA5ZWRjYjY2NTRiNzBmOThjMjdjZThmZTM0Yzg5OGVhNzE3YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjY3ZWZhMmNjMWE2ZWYxMmRiNTllMGQ3YmZmYzUyNmI4MzNjMmY2YzBhMzQ4OTY4YTNiMTFiZmMzNTQzYjYzMWUiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCKJ9asaviqcQ6TdgckIfgSpPDdaZRBvgJPShK7+7ckNXz7BN/UOEUAQd8LX5nOLB0CMQDU0x14ho3fJVYptNjbWjKigo3hOv+r9AQJuul/UTLF8TDqDkBim71rSYd037LUWIE=","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-persistence/BENCHMARK.md b/skills/kumo-model-persistence/BENCHMARK.md index aa524bbbd..a186d5e7e 100644 --- a/skills/kumo-model-persistence/BENCHMARK.md +++ b/skills/kumo-model-persistence/BENCHMARK.md @@ -14,7 +14,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) -- Dataset digest: `sha256:096c403e279883d033c46cf48b03d51e3a6ade8a0c67149a36066124537ddb47` (skill-evaluator-dataset-snapshot/1) +- Dataset digest: `sha256:9ca8b4582ecaaf63896f20a1be87b43525cfdbe62ed58a68d8019d189b0ad61c` (skill-evaluator-dataset-snapshot/1) - Attempts per task: 1 - Environment: `k8s-sandbox` - Tier 2 evidence: required for publication @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 87.6% — baseline ran, but no comparable score was available; uplift unavailable | 79.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 53.3% → 96.7% (+43.4 points) | 36.7% → 73.3% (+36.6 points) | -| Discoverability | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 26.7% → 50.6% (+23.9 points) | 30.6% → 44.5% (+13.9 points) | -| Efficiency | 94.8% — baseline ran, but no comparable score was available; uplift unavailable | 92.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | 82.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 91.7% → 100.0% (+8.3 points) | +| Correctness | 53.3% → 90.0% (+36.7 points) | 63.3% → 86.7% (+23.4 points) | +| Discoverability | 99.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 28.1% → 72.2% (+44.1 points) | 28.9% → 50.6% (+21.7 points) | +| Efficiency | 85.9% — baseline ran, but no comparable score was available; uplift unavailable | 82.6% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 96.9% (-1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.8% → 96.5% (-1.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.2% → 91.9% (+0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 83.5% → 89.9% (+6.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,21 +80,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,095,881 | 752,122 | +343,759 | +45.71% | skill 6/6; base 6/6 | -| claude-code | kpersist-01 | 269,744 | 143,720 | +126,024 | +87.69% | skill 1/1; base 1/1 | -| claude-code | kpersist-02 | 228,074 | 107,402 | +120,672 | +112.36% | skill 1/1; base 1/1 | -| claude-code | kpersist-03 | 230,584 | 143,145 | +87,439 | +61.08% | skill 1/1; base 1/1 | -| claude-code | kpersist-04 | 188,567 | 143,493 | +45,074 | +31.41% | skill 1/1; base 1/1 | -| claude-code | kpersist-05 | 71,305 | 106,857 | -35,552 | -33.27% | skill 1/1; base 1/1 | -| claude-code | kpersist-06 | 107,607 | 107,505 | +102 | +0.09% | skill 1/1; base 1/1 | -| codex | All cases | 398,521 | 311,624 | +86,897 | +27.89% | skill 6/6; base 6/6 | -| codex | kpersist-01 | 77,307 | 72,416 | +4,891 | +6.75% | skill 1/1; base 1/1 | -| codex | kpersist-02 | 42,201 | 13,428 | +28,773 | +214.28% | skill 1/1; base 1/1 | -| codex | kpersist-03 | 98,036 | 61,157 | +36,879 | +60.30% | skill 1/1; base 1/1 | -| codex | kpersist-04 | 92,901 | 62,029 | +30,872 | +49.77% | skill 1/1; base 1/1 | -| codex | kpersist-05 | 46,728 | 61,509 | -14,781 | -24.03% | skill 1/1; base 1/1 | -| codex | kpersist-06 | 41,348 | 41,085 | +263 | +0.64% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,494,402 | 1,063,746 | +430,656 | +40.48% | skill 12/12; base 12/12 | +| claude-code | All cases | 1,178,061 | 689,035 | +489,026 | +70.97% | skill 6/6; base 6/6 | +| claude-code | kpersist-01 | 312,188 | 146,077 | +166,111 | +113.71% | skill 1/1; base 1/1 | +| claude-code | kpersist-02 | 228,280 | 147,904 | +80,376 | +54.34% | skill 1/1; base 1/1 | +| claude-code | kpersist-03 | 228,773 | 72,041 | +156,732 | +217.56% | skill 1/1; base 1/1 | +| claude-code | kpersist-04 | 193,734 | 108,886 | +84,848 | +77.92% | skill 1/1; base 1/1 | +| claude-code | kpersist-05 | 107,344 | 106,819 | +525 | +0.49% | skill 1/1; base 1/1 | +| claude-code | kpersist-06 | 107,742 | 107,308 | +434 | +0.40% | skill 1/1; base 1/1 | +| codex | All cases | 357,551 | 640,372 | -282,821 | -44.17% | skill 6/6; base 6/6 | +| codex | kpersist-01 | 86,040 | 386,406 | -300,366 | -77.73% | skill 1/1; base 1/1 | +| codex | kpersist-02 | 42,472 | 13,606 | +28,866 | +212.16% | skill 1/1; base 1/1 | +| codex | kpersist-03 | 70,384 | 86,760 | -16,376 | -18.88% | skill 1/1; base 1/1 | +| codex | kpersist-04 | 75,695 | 39,061 | +36,634 | +93.79% | skill 1/1; base 1/1 | +| codex | kpersist-05 | 41,998 | 73,582 | -31,584 | -42.92% | skill 1/1; base 1/1 | +| codex | kpersist-06 | 40,962 | 40,957 | +5 | +0.01% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,535,612 | 1,329,407 | +206,205 | +15.51% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-persistence/skill-card.md b/skills/kumo-model-persistence/skill-card.md index 2fe87ed7f..40ba32c38 100644 --- a/skills/kumo-model-persistence/skill-card.md +++ b/skills/kumo-model-persistence/skill-card.md @@ -67,25 +67,21 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
-> ✅ **Overall verdict: PASS — Recommended for publication** - -### Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. - ### Results at a Glance | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 87.6% — baseline ran, but no comparable score was available; uplift unavailable | 79.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 53.3% → 96.7% (+43.4 points) | 36.7% → 73.3% (+36.6 points) | -| Discoverability | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 26.7% → 50.6% (+23.9 points) | 30.6% → 44.5% (+13.9 points) | -| Efficiency | 94.8% — baseline ran, but no comparable score was available; uplift unavailable | 92.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | 82.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 91.7% → 100.0% (+8.3 points) | +| Correctness | 53.3% → 90.0% (+36.7 points) | 63.3% → 86.7% (+23.4 points) | +| Discoverability | 99.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 28.1% → 72.2% (+44.1 points) | 28.9% → 50.6% (+21.7 points) | +| Efficiency | 85.9% — baseline ran, but no comparable score was available; uplift unavailable | 82.6% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. #### Comparison details @@ -95,9 +91,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 96.9% (-1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.8% → 96.5% (-1.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.2% → 91.9% (+0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 83.5% → 89.9% (+6.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -107,8 +103,16 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | +### Overall Verdict + +✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + ## Skill Version(s):
-0.1.0 (source: pyproject.toml; git SHA 7cbda7af, committed 2026-11-18)
+0.1.0 (source: pyproject.toml; git SHA 8aaa6cb3, committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-model-persistence/skill.oms.sig b/skills/kumo-model-persistence/skill.oms.sig index 43ba1bfcf..d48ab63ba 100644 --- a/skills/kumo-model-persistence/skill.oms.sig +++ b/skills/kumo-model-persistence/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1wZXJzaXN0ZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIxNTExYWEwNGM5NWNjYjYwOGMzNGFmMmYxODI0MmE3ZGZmN2EwNGJhOTRkZjZkZDQ4YzE3OWQxMDM5NDM0MmU3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjMzZmMzZDRjM2VkOThiNGQ2NGFhYWQ1YzExYmJlNWYwOGUzY2ViNGU0ZmYzZjRmMWVhYzcxNzliZGJhYjI4YmUiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiYjFlODFlNjg4NmQzZGE3M2ZjMDMyNmY4ZjBkNWE2M2EwYzcyMzY5ZDZmMjVmZDNjNDI5MjdlOWFlMjAwZWUyOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjM2N2U4NzFhNzJlM2U2YzdiNTFkYWI5YTMyYjMzMDk0MmQwMDI5ZWU2NDczYzMzNzY3MjMyNjRkMjJmM2NhNmEiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9yZXRlbnRpb24ubWQiLAogICAgICAgICJkaWdlc3QiOiAiMmIwNTRhZjMzZjgxNjJjNTE4ZGVhOWY3YWQ4NDRjMGQ0MTYwN2IwMDNkNTgyMWQ1MGEyYmY2OGJlNmNhY2YzMiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3JldXNlLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjJjNWE1MDMxYzg1MDE5ZjEyMTVlNmM3NDBkY2VkZTdhNmU5MjA4NTJiM2Y1NzJmMDRmOTRkYTBlY2NjOGFlODEiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZXRhaW5fYXJ0aWZhY3RzLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjRkM2QwM2IxMDg5YTAyNDBkMWY5MTNjMWVkMmY4ZTRhZjk4OWVjYjhkM2UxOTljMzJmZGRhZTkwZmNiYWFkM2IiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI5MzRlMTRiZWJkMmFiNTMyMmM3YmFmYmZhNmJkNzFlZWQ2MmZmYjNlOTAyMDZiMGQxZDYwOTI5MGM4NjU5OWY5IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMHTQCjGe5J4FqdfHSg9Kctz2/ss2/F0xkiHXwdNzezDoHD48Wswkt30med75qhWgawIwNddwEMf5RoLTNgn388Ri89neGNK58++zqSxTebFj8sya/BOKuBIPpISR1izIEHlL","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1wZXJzaXN0ZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJiMmRmMTZhMjIxMTAxZWQ4N2JkZDYwOTdmMTk1ZTUxYzJlMmVmZmEwN2NlNjRhMThhNjliM2VkMmYzMDNiZDJhIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2RkNjg1YzhlNmZjMDg1ODdhN2FhNWZkOWQzNjhhYTNhMDhjNTQ4M2FhZWQ0YTliNzBjZWE1MTE4Y2ExMzUzYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJiMWU4MWU2ODg2ZDNkYTczZmMwMzI2ZjhmMGQ1YTYzYTBjNzIzNjlkNmYyNWZkM2M0MjkyN2U5YWUyMDBlZTI4IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiM2JiZjZmNDg1OGNmNGEwMWY4NTAzMThjNTliNThmNDUzMTkwNzQ3NTI1OTc4ZGI2ZjIxYTA2MjRlOTBiYmRlZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3JldGVudGlvbi5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIyYjA1NGFmMzNmODE2MmM1MThkZWE5ZjdhZDg0NGMwZDQxNjA3YjAwM2Q1ODIxZDUwYTJiZjY4YmU2Y2FjZjMyIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcmV1c2UubWQiLAogICAgICAgICJkaWdlc3QiOiAiMmM1YTUwMzFjODUwMTlmMTIxNWU2Yzc0MGRjZWRlN2E2ZTkyMDg1MmIzZjU3MmYwNGY5NGRhMGVjY2M4YWU4MSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JldGFpbl9hcnRpZmFjdHMucHkiLAogICAgICAgICJkaWdlc3QiOiAiNGQzZDAzYjEwODlhMDI0MGQxZjkxM2MxZWQyZjhlNGFmOTg5ZWNiOGQzZTE5OWMzMmZkZGFlOTBmY2JhYWQzYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogImU0OThkNjY3MmIxNmM5NDA0NTdiZTVjYzU0NzljYzkxYzM4MzNhOTNjY2U5ZTJkOTE3ZDBkZjIzYzdlOTdmN2MiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDSbrAfQsoVPG4DRY9eYDMrvhlYQdA3cSL9C2wNpHjqjJOhwgYcJp+Qa3WoZW8hVSoCMQCQGVN4ym7lKLj+ruqEF266SAdrnF983/rGCZ7h8z+PaSl+VHHpNPk1Pelyu9TZNk4=","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-tuning/BENCHMARK.md b/skills/kumo-model-tuning/BENCHMARK.md index 5bfb28313..c67ee01e1 100644 --- a/skills/kumo-model-tuning/BENCHMARK.md +++ b/skills/kumo-model-tuning/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 98.9% — baseline ran, but no comparable score was available; uplift unavailable | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 99.7% — baseline ran, but no comparable score was available; uplift unavailable | 92.2% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 76.0% → 100.0% (+24.0 points) | 60.0% → 96.0% (+36.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 60.0% → 96.0% (+36.0 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 62.5% → 100.0% (+37.5 points) | 51.7% → 70.8% (+19.1 points) | -| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 87.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 52.5% → 100.0% (+47.5 points) | 51.7% → 80.8% (+29.1 points) | +| Efficiency | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 91.8% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.9% → 97.4% (-0.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 97.3% (-0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.6% → 91.9% (+0.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 91.5% → 91.9% (+0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 734,363 | 830,668 | -96,305 | -11.59% | skill 5/5; base 5/5 | -| claude-code | ktune-01 | 109,059 | 71,625 | +37,434 | +52.26% | skill 1/1; base 1/1 | -| claude-code | ktune-02 | 146,360 | 218,578 | -72,218 | -33.04% | skill 1/1; base 1/1 | -| claude-code | ktune-03 | 148,102 | 141,918 | +6,184 | +4.36% | skill 1/1; base 1/1 | -| claude-code | ktune-04 | 109,094 | 143,000 | -33,906 | -23.71% | skill 1/1; base 1/1 | -| claude-code | ktune-05 | 221,748 | 255,547 | -33,799 | -13.23% | skill 1/1; base 1/1 | -| codex | All cases | 253,211 | 239,042 | +14,169 | +5.93% | skill 5/5; base 5/5 | -| codex | ktune-01 | 46,169 | 49,399 | -3,230 | -6.54% | skill 1/1; base 1/1 | -| codex | ktune-02 | 72,379 | 54,978 | +17,401 | +31.65% | skill 1/1; base 1/1 | -| codex | ktune-03 | 56,882 | 40,860 | +16,022 | +39.21% | skill 1/1; base 1/1 | -| codex | ktune-04 | 45,835 | 48,842 | -3,007 | -6.16% | skill 1/1; base 1/1 | -| codex | ktune-05 | 31,946 | 44,963 | -13,017 | -28.95% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 987,574 | 1,069,710 | -82,136 | -7.68% | skill 10/10; base 10/10 | +| claude-code | All cases | 922,430 | 680,648 | +241,782 | +35.52% | skill 5/5; base 5/5 | +| claude-code | ktune-01 | 108,911 | 71,704 | +37,207 | +51.89% | skill 1/1; base 1/1 | +| claude-code | ktune-02 | 148,004 | 178,646 | -30,642 | -17.15% | skill 1/1; base 1/1 | +| claude-code | ktune-03 | 148,059 | 142,151 | +5,908 | +4.16% | skill 1/1; base 1/1 | +| claude-code | ktune-04 | 108,499 | 107,619 | +880 | +0.82% | skill 1/1; base 1/1 | +| claude-code | ktune-05 | 408,957 | 180,528 | +228,429 | +126.53% | skill 1/1; base 1/1 | +| codex | All cases | 265,904 | 280,821 | -14,917 | -5.31% | skill 5/5; base 5/5 | +| codex | ktune-01 | 41,800 | 63,362 | -21,562 | -34.03% | skill 1/1; base 1/1 | +| codex | ktune-02 | 56,900 | 54,890 | +2,010 | +3.66% | skill 1/1; base 1/1 | +| codex | ktune-03 | 56,919 | 54,763 | +2,156 | +3.94% | skill 1/1; base 1/1 | +| codex | ktune-04 | 46,208 | 62,868 | -16,660 | -26.50% | skill 1/1; base 1/1 | +| codex | ktune-05 | 64,077 | 44,938 | +19,139 | +42.59% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,188,334 | 961,469 | +226,865 | +23.60% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-tuning/skill-card.md b/skills/kumo-model-tuning/skill-card.md index adee29b54..357412f57 100644 --- a/skills/kumo-model-tuning/skill-card.md +++ b/skills/kumo-model-tuning/skill-card.md @@ -73,14 +73,18 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 98.9% — baseline ran, but no comparable score was available; uplift unavailable | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 99.7% — baseline ran, but no comparable score was available; uplift unavailable | 92.2% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 76.0% → 100.0% (+24.0 points) | 60.0% → 96.0% (+36.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 60.0% → 96.0% (+36.0 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 62.5% → 100.0% (+37.5 points) | 51.7% → 70.8% (+19.1 points) | -| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 87.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 52.5% → 100.0% (+47.5 points) | 51.7% → 80.8% (+29.1 points) | +| Efficiency | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 91.8% — baseline ran, but no comparable score was available; uplift unavailable | -Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. #### Comparison details @@ -89,9 +93,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.9% → 97.4% (-0.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 97.3% (-0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.6% → 91.9% (+0.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 91.5% → 91.9% (+0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
0.1.0 (source: pyproject.toml)
diff --git a/skills/kumo-model-tuning/skill.oms.sig b/skills/kumo-model-tuning/skill.oms.sig index e283985cc..0cda032e5 100644 --- a/skills/kumo-model-tuning/skill.oms.sig +++ b/skills/kumo-model-tuning/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC10dW5pbmciLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNzMwMzdhYTY5NDE0NmNlNWRjNDhjMmEwOGNiYjIyYWQ5ZjllZDMwMTBkYmFhNWMxY2Q2MGNkNWNkMDg4NzIyYyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYjg5OWUxN2Y3OGM5Y2NmZjgzNDRlMmNlZmQxYTdmYTU2OGQ5ZjQ4NmJiNzYwYTRhZjUxOGRiMzQ2NDMxMDE3NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxNjQwMWRiY2Y5OTM0ZTY1MTkyNDdhNDNiN2VkYTYxNzc5NjRiMTJmNzBiNjg2MTY0MWU0NDE2MGU2ZDhlN2FiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNThiNmNmMzIyNDI1M2VlZTVmOWMxZTAzYjFmODJlM2E5NWVlOGYxYzUyN2MyNWNiYjM1YzE3NTQ4OTBkZGZlYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcGxhbnMubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjNlYjQyYjBmODk1M2QxNzAwMjU1OGQwNWE4ODI4NTE5MzFkOGNiNTEwY2E4ZjdlNWM2MGViOWNmNjc5MDJjNGQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkNjA1YWM0ZmYwOWExM2JiNDI4ZDRlNjE1NzNjYzA5MjQzMDE3N2UxZGY5YThiMWVkODYyNzc5OTExZDkyOWNhIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIKICAgICAgXSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCZ6bMenexf57YpQvEo5xKzIBIHwLJe7H49gkCHN8uVLJrQy5reHmerjfzhtZvX2X0CMQD4aD59e9Sq939kwAwMV+tYdFN/beIui0prh/cCDT21Dvih64JSH7FlBTHWvqfKF1o=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC10dW5pbmciLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNjM1NzVkY2UzNTY2NzEzMGE1NTY1MjYyZDZlZmU1NDAyYWUxODFhNWUwOTVjM2NmMDM0MzE0YjQ5MjdhYTk0ZSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiM2I5MTRmN2ZiYTM5MDgyNDdiNDAwYjNjYzdkOWZmZjA1MGNkZTE5OTZiYmUyNWNkZmVkNTA3NjdhOTYwMjk2NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMTY0MDFkYmNmOTkzNGU2NTE5MjQ3YTQzYjdlZGE2MTc3OTY0YjEyZjcwYjY4NjE2NDFlNDQxNjBlNmQ4ZTdhYiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI1OGI2Y2YzMjI0MjUzZWVlNWY5YzFlMDNiMWY4MmUzYTk1ZWU4ZjFjNTI3YzI1Y2JiMzVjMTc1NDg5MGRkZmViIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiM2ViNDJiMGY4OTUzZDE3MDAyNTU4ZDA1YTg4Mjg1MTkzMWQ4Y2I1MTBjYThmN2U1YzYwZWI5Y2Y2NzkwMmM0ZCIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9wbGFucy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjE0MjhkZGQwYWIxZmE2MTBlOTFiZDFlMTg5YzE4Y2IzNmIyZDg3MjU2ZWIwYjg0YTU5YmFlZGViOTc5MjFhMjciLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCpPh9gERO0P1E4aX3WIkFPrXnsuPRDfZW6NGkZIo2nhyKiRT2l8aYjlTGJlDn0I/oCMQD61QYNU0U+6PcSubdDyV745hDnk19R+PmB1l2+B4R0d1ceSVaqIgZKJ/BBd6ytmRI=","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-training-diagnostics/BENCHMARK.md b/skills/kumo-training-diagnostics/BENCHMARK.md index 4a5a316a6..377036542 100644 --- a/skills/kumo-training-diagnostics/BENCHMARK.md +++ b/skills/kumo-training-diagnostics/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 78.2% — baseline ran, but no comparable score was available; uplift unavailable | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | -| Correctness | 48.0% → 80.0% (+32.0 points) | 32.0% → 88.0% (+56.0 points) | -| Discoverability | 75.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 38.2% → 45.7% (+7.5 points) | 36.8% → 65.2% (+28.4 points) | -| Efficiency | 90.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 83.1% — baseline ran, but no comparable score was available; uplift unavailable | 88.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 48.0% → 80.0% (+32.0 points) | 32.0% → 84.0% (+52.0 points) | +| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 38.2% → 56.0% (+17.8 points) | 40.2% → 68.5% (+28.3 points) | +| Efficiency | 94.3% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.5% → 96.9% (-1.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.5% → 97.0% (-1.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.9% → 91.9% (+1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 92.1% → 91.9% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 715,359 | 535,886 | +179,473 | +33.49% | skill 5/5; base 5/5 | -| claude-code | kdiag-01 | 155,528 | 142,961 | +12,567 | +8.79% | skill 1/1; base 1/1 | -| claude-code | kdiag-02 | 197,199 | 71,171 | +126,028 | +177.08% | skill 1/1; base 1/1 | -| claude-code | kdiag-03 | 146,734 | 107,060 | +39,674 | +37.06% | skill 1/1; base 1/1 | -| claude-code | kdiag-04 | 108,287 | 143,366 | -35,079 | -24.47% | skill 1/1; base 1/1 | -| claude-code | kdiag-05 | 107,611 | 71,328 | +36,283 | +50.87% | skill 1/1; base 1/1 | -| codex | All cases | 291,440 | 252,141 | +39,299 | +15.59% | skill 5/5; base 5/5 | -| codex | kdiag-01 | 61,309 | 57,259 | +4,050 | +7.07% | skill 1/1; base 1/1 | -| codex | kdiag-02 | 78,558 | 77,086 | +1,472 | +1.91% | skill 1/1; base 1/1 | -| codex | kdiag-03 | 61,614 | 54,765 | +6,849 | +12.51% | skill 1/1; base 1/1 | -| codex | kdiag-04 | 44,357 | 45,264 | -907 | -2.00% | skill 1/1; base 1/1 | -| codex | kdiag-05 | 45,602 | 17,767 | +27,835 | +156.67% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,006,799 | 788,027 | +218,772 | +27.76% | skill 10/10; base 10/10 | +| claude-code | All cases | 758,267 | 536,081 | +222,186 | +41.45% | skill 5/5; base 5/5 | +| claude-code | kdiag-01 | 197,050 | 142,886 | +54,164 | +37.91% | skill 1/1; base 1/1 | +| claude-code | kdiag-02 | 197,230 | 107,354 | +89,876 | +83.72% | skill 1/1; base 1/1 | +| claude-code | kdiag-03 | 148,917 | 107,162 | +41,755 | +38.96% | skill 1/1; base 1/1 | +| claude-code | kdiag-04 | 107,579 | 107,407 | +172 | +0.16% | skill 1/1; base 1/1 | +| claude-code | kdiag-05 | 107,491 | 71,272 | +36,219 | +50.82% | skill 1/1; base 1/1 | +| codex | All cases | 274,654 | 179,075 | +95,579 | +53.37% | skill 5/5; base 5/5 | +| codex | kdiag-01 | 61,263 | 27,057 | +34,206 | +126.42% | skill 1/1; base 1/1 | +| codex | kdiag-02 | 62,133 | 40,982 | +21,151 | +51.61% | skill 1/1; base 1/1 | +| codex | kdiag-03 | 61,343 | 46,850 | +14,493 | +30.93% | skill 1/1; base 1/1 | +| codex | kdiag-04 | 44,384 | 46,477 | -2,093 | -4.50% | skill 1/1; base 1/1 | +| codex | kdiag-05 | 45,531 | 17,709 | +27,822 | +157.11% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,032,921 | 715,156 | +317,765 | +44.43% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-training-diagnostics/skill-card.md b/skills/kumo-training-diagnostics/skill-card.md index 7c5f6ea09..530c205ea 100644 --- a/skills/kumo-training-diagnostics/skill-card.md +++ b/skills/kumo-training-diagnostics/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Evaluate an existing Kumo training run in a Python notebook using aggregate metrics, trial results, learning curves, refit evidence, and training progress.
+Evaluate an existing Kumo training run in a Python notebook using aggregate metrics, trial results, learning curves, refit evidence, and training progress; use for model-quality questions or training diagnostics, inspecting existing results without automatically tuning, rerunning, or deploying a model.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill with coding agents to answer how a Kumo model performed, which trial was selected, and what its learning curves and training progress show, without tuning, rerunning, or deploying a model.
+Developers and data scientists use this skill to assess an existing Kumo training run in a notebook — reviewing available metrics, trial selection evidence, learning curves, refit results, and training progress — and to return an evidence-backed quality assessment without retraining or tuning.
### Deployment Geography for Use:
Global
@@ -25,15 +25,16 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/diagnostics.md](references/diagnostics.md)
-- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
+- [Diagnostic workflow](references/diagnostics.md)
+- [Kumo model persistence skill](../kumo-model-persistence/SKILL.md)
+- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
## Skill Output:
**Output Type(s):** [Analysis, Code]
-**Output Format:** [Markdown assessment with inline Python code blocks]
+**Output Format:** [Markdown with inline Python code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Inspection only; does not tune, rerun, or deploy a model.]
+**Other Properties Related to Output:** [None]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -42,7 +43,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated `k8s-sandbox` pod; each task was also run without the skill as a baseline. Evaluator version `1.6.10`, evaluated 2026-11-18.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated `k8s-sandbox` pod; each task was also run without the skill as a baseline. Three-tier evaluation: Tier 1 static validation, Tier 2 semantic deduplication, Tier 3 live agent evaluation.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -64,28 +65,22 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
-**Overall verdict: PASS — Recommended for publication** +**Overall verdict:** ✅ PASS — Recommended for publication -### Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. ### Results at a Glance | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 78.2% — baseline ran, but no comparable score was available; uplift unavailable | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | -| Correctness | 48.0% → 80.0% (+32.0 points) | 32.0% → 88.0% (+56.0 points) | -| Discoverability | 75.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 38.2% → 45.7% (+7.5 points) | 36.8% → 65.2% (+28.4 points) | -| Efficiency | 90.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | - -**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. - -Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. +| Overall | 83.1% — baseline ran, but no comparable score was available; uplift unavailable | 88.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 48.0% → 80.0% (+32.0 points) | 32.0% → 84.0% (+52.0 points) | +| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 38.2% → 56.0% (+17.8 points) | 40.2% → 68.5% (+28.3 points) | +| Efficiency | 94.3% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | -A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. ### Comparison details @@ -94,20 +89,12 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.5% → 96.9% (-1.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.5% → 97.0% (-1.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.9% → 91.9% (+1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | - -### Tier Status - -| Tier | Purpose | Status | Evidence | -|---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 12 finding(s) | -| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | +| codex | Efficiency | Token-only comparison | 92.1% → 91.9% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
-7cbda7af (source: git SHA, committed 2026-11-18)
+0.1.0 (source: pyproject.toml)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-training-diagnostics/skill.oms.sig b/skills/kumo-training-diagnostics/skill.oms.sig index e3b00f15b..89ca57bf4 100644 --- a/skills/kumo-training-diagnostics/skill.oms.sig +++ b/skills/kumo-training-diagnostics/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby10cmFpbmluZy1kaWFnbm9zdGljcyIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI3MGQ5YWJmOWQyMGU4YzBmYjYzYmFlNmI4MWMyYThlYjMzZWFmMzg0YWI4MjUwNDVjZTkxOGNjYWMxY2VhMGJlIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxZWYwMWYzM2E2ZDhmNWEzM2NiNDIyZTc5NzNkYzM2NWM3OWU5YjI1YzkzMTFiYjFjYzAwZjVlYzhkZjQ5YzYxIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjY0ZGQ4OGFlNjYwMmZiMWYyN2E0ZjQzYTRlNDJkNjA3ZjczZmQ5ZjE3YWE0ODgxNWEzOGNlOWY0MzQwZmVmNzYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI4YzRjN2IzYTY5ZGQxNDcxYTAzYjI1NzNiN2JlZTE5NDExMjliN2E1MmM4OTk4ZWI4OTc3MjJkZjIxNTUyOTU0IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9kaWFnbm9zdGljcy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZTc1ZDMwZTU0NTUzMjJmODVjNTQwZDhiNDM3ZGIwYzAwNzYxZDUyNzU4YTUyZjNiOWE4YTc2ODAzNjFmY2UzMCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjlhMjllMzE3NzM5ODU2OTQxZDczNzI3MzE5YTQzNmI2YjBhYWRkYTNjNDk3MjAwYWI5N2E1OGZkZGUyZGJhZTMiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIKICAgICAgXSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMC43nABz4HhURsMYsNFBWXibb9nEu1EYKoWd1qD0up+Fe2KTQWbMoiPp60kpy7OW+AIxAOfCkMC7Zyenk3Yty7yK8IJZGT4bi7hxoWVGzhkVBciwg6t4305PksMMU59M/gv0AQ==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby10cmFpbmluZy1kaWFnbm9zdGljcyIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIwMjJkMjkxYzMwYTkzMGE1YTkyMjhhOWY0Yzk2MTZjZTlhMjIxY2Y2M2NkYzY3NjgyNzAwNWM1MjNhYjc0NzU0IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJkMDljYjdiNGY4MGRjNTdiZjEwOTY3MDYxNTJkNTQ4N2YyZGExYWRhNDdiYjY3OGQxYjg2MzI1NTlkN2JmYzA3IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI2NGRkODhhZTY2MDJmYjFmMjdhNGY0M2E0ZTQyZDYwN2Y3M2ZkOWYxN2FhNDg4MTVhMzhjZTlmNDM0MGZlZjc2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjhjNGM3YjNhNjlkZDE0NzFhMDNiMjU3M2I3YmVlMTk0MTEyOWI3YTUyYzg5OThlYjg5NzcyMmRmMjE1NTI5NTQiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlNzVkMzBlNTQ1NTMyMmY4NWM1NDBkOGI0MzdkYjBjMDA3NjFkNTI3NThhNTJmM2I5YThhNzY4MDM2MWZjZTMwIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2RpYWdub3N0aWNzLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMWIwNTQ3MzBjNjkyNTJjYzU5ZjFkM2M4ZDhmNDA3MDYwYmNmYTI1NDk3N2QyZmViNTA3MWNmY2E5ZDVjNzc1YiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMFhuql36zE6dgiSvxKKu6C8432nO2F4DH1bIAHf5V1Xz0Ad/1AzTyPx/ECsF+X7mLAIxAL7fIQb6woGh39UigiSihRZ/HP644Q7519C7gqqiJEasBZnWIcQsY6bWNvDyxBkVGA==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumoai-reference/BENCHMARK.md b/skills/kumoai-reference/BENCHMARK.md index db1b6eedd..800ab41ba 100644 --- a/skills/kumoai-reference/BENCHMARK.md +++ b/skills/kumoai-reference/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.6% — baseline ran, but no comparable score was available; uplift unavailable | 89.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 80.0% → 100.0% (+20.0 points) | 90.0% → 90.0% (±0.0 points) | -| Correctness | 68.0% → 100.0% (+32.0 points) | 92.0% → 100.0% (+8.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 44.9% → 84.0% (+39.1 points) | 77.8% → 80.4% (+2.6 points) | -| Efficiency | 83.8% — baseline ran, but no comparable score was available; uplift unavailable | 84.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 93.2% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 90.0% → 80.0% (-10.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 84.0% → 96.0% (+12.0 points) | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 88.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 50.5% → 78.0% (+27.5 points) | 58.0% → 84.7% (+26.7 points) | +| Efficiency | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | 85.4% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.2% → 81.5% (-15.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 94.3% → 87.8% (-6.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 77.6% → 87.0% (+9.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 77.4% → 87.0% (+9.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,477,453 | 598,733 | +878,720 | +146.76% | skill 5/5; base 5/5 | -| claude-code | kref-01 | 121,691 | 35,737 | +85,954 | +240.52% | skill 1/1; base 1/1 | -| claude-code | kref-02 | 445,404 | 148,446 | +296,958 | +200.04% | skill 1/1; base 1/1 | -| claude-code | kref-03 | 134,125 | 108,917 | +25,208 | +23.14% | skill 1/1; base 1/1 | -| claude-code | kref-04 | 250,746 | 38,242 | +212,504 | +555.68% | skill 1/1; base 1/1 | -| claude-code | kref-05 | 525,487 | 267,391 | +258,096 | +96.52% | skill 1/1; base 1/1 | -| codex | All cases | 751,517 | 759,151 | -7,634 | -1.01% | skill 5/5; base 5/5 | -| codex | kref-01 | 65,056 | 54,410 | +10,646 | +19.57% | skill 1/1; base 1/1 | -| codex | kref-02 | 183,823 | 190,612 | -6,789 | -3.56% | skill 1/1; base 1/1 | -| codex | kref-03 | 121,056 | 90,774 | +30,282 | +33.36% | skill 1/1; base 1/1 | -| codex | kref-04 | 142,837 | 47,313 | +95,524 | +201.90% | skill 1/1; base 1/1 | -| codex | kref-05 | 238,745 | 376,042 | -137,297 | -36.51% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 2,228,970 | 1,357,884 | +871,086 | +64.15% | skill 10/10; base 10/10 | +| claude-code | All cases | 1,351,652 | 1,272,515 | +79,137 | +6.22% | skill 5/5; base 5/5 | +| claude-code | kref-01 | 121,660 | 35,613 | +86,047 | +241.62% | skill 1/1; base 1/1 | +| claude-code | kref-02 | 523,009 | 148,685 | +374,324 | +251.76% | skill 1/1; base 1/1 | +| claude-code | kref-03 | 233,277 | 306,293 | -73,016 | -23.84% | skill 1/1; base 1/1 | +| claude-code | kref-04 | 325,840 | 109,513 | +216,327 | +197.54% | skill 1/1; base 1/1 | +| claude-code | kref-05 | 147,866 | 672,411 | -524,545 | -78.01% | skill 1/1; base 1/1 | +| codex | All cases | 825,268 | 673,781 | +151,487 | +22.48% | skill 5/5; base 5/5 | +| codex | kref-01 | 69,115 | 28,809 | +40,306 | +139.91% | skill 1/1; base 1/1 | +| codex | kref-02 | 280,824 | 252,624 | +28,200 | +11.16% | skill 1/1; base 1/1 | +| codex | kref-03 | 120,693 | 91,963 | +28,730 | +31.24% | skill 1/1; base 1/1 | +| codex | kref-04 | 137,479 | 47,353 | +90,126 | +190.33% | skill 1/1; base 1/1 | +| codex | kref-05 | 217,157 | 253,032 | -35,875 | -14.18% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,176,920 | 1,946,296 | +230,624 | +11.85% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumoai-reference/skill-card.md b/skills/kumoai-reference/skill-card.md index 068961f4c..4645fceff 100644 --- a/skills/kumoai-reference/skill-card.md +++ b/skills/kumoai-reference/skill-card.md @@ -25,12 +25,12 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/graph_construction.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/graph_construction.md)
-- [references/pql_writing.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/pql_writing.md)
-- [references/kumo_finetuning_sdk.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/kumo_finetuning_sdk.md)
-- [references/task_journeys.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/task_journeys.md)
-- [references/outputs.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/outputs.md)
-- [references/troubleshooting_signals.md](https://github.com/NVIDIA/kumo-engine/blob/main/skills/kumoai-reference/references/troubleshooting_signals.md)
+- [references/graph_construction.md](references/graph_construction.md)
+- [references/pql_writing.md](references/pql_writing.md)
+- [references/kumo_finetuning_sdk.md](references/kumo_finetuning_sdk.md)
+- [references/task_journeys.md](references/task_journeys.md)
+- [references/outputs.md](references/outputs.md)
+- [references/troubleshooting_signals.md](references/troubleshooting_signals.md)
- [NVIDIA/kumo-engine#134 (graph visualization)](https://github.com/NVIDIA/kumo-engine/pull/134)
- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
@@ -80,12 +80,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.6% — baseline ran, but no comparable score was available; uplift unavailable | 89.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 80.0% → 100.0% (+20.0 points) | 90.0% → 90.0% (±0.0 points) | -| Correctness | 68.0% → 100.0% (+32.0 points) | 92.0% → 100.0% (+8.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 44.9% → 84.0% (+39.1 points) | 77.8% → 80.4% (+2.6 points) | -| Efficiency | 83.8% — baseline ran, but no comparable score was available; uplift unavailable | 84.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 93.2% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 90.0% → 80.0% (-10.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 84.0% → 96.0% (+12.0 points) | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 88.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 50.5% → 78.0% (+27.5 points) | 58.0% → 84.7% (+26.7 points) | +| Efficiency | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | 85.4% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -100,9 +100,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.2% → 81.5% (-15.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 94.3% → 87.8% (-6.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 77.6% → 87.0% (+9.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 77.4% → 87.0% (+9.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -113,7 +113,7 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | ## Skill Version(s):
-0.1.0 (source: pyproject.toml; git SHA 7cbda7af, committed 2026-11-18)
+0.1.0 (source: pyproject.toml; git SHA 8aaa6cb3, committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumoai-reference/skill.oms.sig b/skills/kumoai-reference/skill.oms.sig index e3a35edbd..55a39a66e 100644 --- a/skills/kumoai-reference/skill.oms.sig +++ b/skills/kumoai-reference/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtb2FpLXJlZmVyZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJhY2YzZDk3ZDdlNWZlYWVjMTlhYjJlMzFjYmJjNjgxYzUzMzVlZDVkODczNDdkMmVkMGFjN2I5NWZkYjIxYmQ3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGh1YiIKICAgICAgXSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiNmIxMzhmMjg2NmJjNDU3MjVjNzNkZmUxODdlMmU2YmJiZjQ0NDE0MDU5ODMwMGQ5MDAyNGMwNGYxZWUzMDAzYSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIwNDMxNjJiYWJjNzU1YWJhMTM1ODI1YWYzY2EzNzVkOGYwZDUxZDkwNGNkY2FiZTYzZjczOWE2M2Y4NGNkNGFmIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiZTgwY2FjYWNjYzYwN2IyMDAxZmMxOTA0NGRjZDcxMDkyYjVhMDU0M2JmNTc2MzAyZGZkZjA4MTJmNDc4ZGU2YyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2dyYXBoX2NvbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI4ZTg4ZTIzYjE3YTIwZTE3YTkyNzg2ODc0MmZjNDA1MDY0YTg4ZDE0NWYyY2I4ODM2YTg1MmUzNDAwNWU0YmJjIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMva3Vtb19maW5ldHVuaW5nX3Nkay5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJlMDQxYWY1ZTkyZmI2NmYyMjE1NWU0NTk0Mjk5NmFlZjM4ZWFjNGE0Y2E3ZDE0YjlkOWNhZWU4ZWFmNmI0OWI4IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvb3V0cHV0cy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJmMjlmM2QyZDJiY2E5NTUwZGU1MjViYjViYzYyZTU1ODc4ZTBmNTRiYTdmNWYxMDliMDJmYzk3OGJiYjg1ZWJhIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcHFsX3dyaXRpbmcubWQiLAogICAgICAgICJkaWdlc3QiOiAiNDU3ODdkYWNiOGU5NzcyYWU5NWIwNTc0ZjI5NzgzNDg3ZTRjY2M5YjcwYWVlMWM5MDRkODBlZDJlODVjMTNhZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3Rhc2tfam91cm5leXMubWQiLAogICAgICAgICJkaWdlc3QiOiAiYzA5YzIzYmExMGM3YWEwMDg5OTY0NjAwMGExNmVkYjMyYzg5ZDI5YzYzNTllNjE0ZGM5YWM2YTJlYTkyMGYwOCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3Ryb3VibGVzaG9vdGluZ19zaWduYWxzLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjc5YjU1NWRiZjJkNDdjMjk4ZWNlNjRmNGY1ZDg4NGRlNzdkNGY3NmE2NTNiYTllMjE4MzMwYzNmMWMxNDdmYjUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJhNDBiYTQ3NDBmOWU2ZThlMTA5MTEwMjFmZGQ4NDBiMTc4NzdlYjU0NzUwZGUwNWUzMWVmYTBhZTMxZjgzNWRjIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQD1KW//rd8bZMnnOxeBxX4wR/iDo/3dejv0QP21FzIC2FXpIXjKg8q0iZjTqZ5xetYCMQCDq5Z6IQ3T/bxSRXcEGAjw49Bkuqn+y5/I2WoOzIbKs62b8ptNxYJ2WFsIwV2An0Y=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtb2FpLXJlZmVyZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI3YmMzMjU5ODMxYjdhODViOGI2YzhmYWQ0YjU4OWNjNDkxYzczODUwYzdiZmFjMGQ3ZTZkZmFmZmJmY2FiNzY2IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzEwMjczMjkxNWMwMGQ0MzhjNTFhMDUxZGRiMWEzY2U2ZTZhM2RmNjA3ZGFmZDI1MDk4ZWIwYTAwMjdiNzAzOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMDQzMTYyYmFiYzc1NWFiYTEzNTgyNWFmM2NhMzc1ZDhmMGQ1MWQ5MDRjZGNhYmU2M2Y3MzlhNjNmODRjZDRhZiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlODBjYWNhY2NjNjA3YjIwMDFmYzE5MDQ0ZGNkNzEwOTJiNWEwNTQzYmY1NzYzMDJkZmRmMDgxMmY0NzhkZTZjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOGU4OGUyM2IxN2EyMGUxN2E5Mjc4Njg3NDJmYzQwNTA2NGE4OGQxNDVmMmNiODgzNmE4NTJlMzQwMDVlNGJiYyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvZ3JhcGhfY29uc3RydWN0aW9uLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlMDQxYWY1ZTkyZmI2NmYyMjE1NWU0NTk0Mjk5NmFlZjM4ZWFjNGE0Y2E3ZDE0YjlkOWNhZWU4ZWFmNmI0OWI4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9rdW1vX2ZpbmV0dW5pbmdfc2RrLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJmMjlmM2QyZDJiY2E5NTUwZGU1MjViYjViYzYyZTU1ODc4ZTBmNTRiYTdmNWYxMDliMDJmYzk3OGJiYjg1ZWJhIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9vdXRwdXRzLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI0NTc4N2RhY2I4ZTk3NzJhZTk1YjA1NzRmMjk3ODM0ODdlNGNjYzliNzBhZWUxYzkwNGQ4MGVkMmU4NWMxM2FmIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9wcWxfd3JpdGluZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzA5YzIzYmExMGM3YWEwMDg5OTY0NjAwMGExNmVkYjMyYzg5ZDI5YzYzNTllNjE0ZGM5YWM2YTJlYTkyMGYwOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdGFza19qb3VybmV5cy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNzliNTU1ZGJmMmQ0N2MyOThlY2U2NGY0ZjVkODg0ZGU3N2Q0Zjc2YTY1M2JhOWUyMTgzMzBjM2YxYzE0N2ZiNSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdHJvdWJsZXNob290aW5nX3NpZ25hbHMubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjA3OTAyMjU1ZDhlNTIwYzc0ZmMxOThmZTBhZWQzMzEyNGJmNDhhYWI5MTlkOWEzZTZkZmI5OTY3YjdkMjU0YzMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMGj8hOUtRh5Vg8qQTMqDM6OX/FAA0TaoIvkVdpKr88BMB02os0wVr9SXrGR0QfXfXAIwC/VDydWUPVbxU3fz4sfOkcw0G5h7SGynZ7KOznRD5fn12P75DJlrrekf+LCh3g1y","keyid":""}]}} \ No newline at end of file From bfbf737f24e01287928b3ffd4ed705b1530349a7 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Wed, 18 Nov 2026 17:44:28 -0600 Subject: [PATCH 32/35] agent-skills: read a CSV time column as text only for a format Arrow parses Reading every CSV time column with a declared format as text broke two cases: an empty cell became '' and failed to parse, and a chrono-only format (%.f, %:z) fell back to a cast to naive timestamp[us], losing the nanoseconds or UTC offset the reader used to infer. The column is now read as text, with '' as null, only when Arrow can parse the format; a chrono-only format reads and casts as without one. Co-Authored-By: Claude Opus 5.5 --- agent-skills/skills/hosts/generic/runtime.md | 30 +++++++++++-------- skills/kumo-data-science/SKILL.md | 2 +- .../hosts/generic/runtime.md | 30 +++++++++++-------- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/agent-skills/skills/hosts/generic/runtime.md b/agent-skills/skills/hosts/generic/runtime.md index 683034bb9..558400fc9 100644 --- a/agent-skills/skills/hosts/generic/runtime.md +++ b/agent-skills/skills/hosts/generic/runtime.md @@ -315,6 +315,7 @@ system; the destination may be in another system (`adapter.md` §4) and plays no ```python import os + import re import pyarrow as pa import pyarrow.compute as pc import pyarrow.csv @@ -334,10 +335,14 @@ system; the destination may be in another system (`adapter.md` §4) and plays no if ENTITY_TIME: # the entity Table's time_column, else None keep[ENTITY_TIME] = 'TIMESTAMP' source_fs, source_path = arrow_fs(ENTITY_SOURCE) + # Arrow parses C-style directives only; chrono's own (%.f, %-m, %:z, %3f, %f) it cannot. + ARROW_FORMAT = (ENTITY_TIME_FORMAT + if ENTITY_TIME_FORMAT and not re.search(r'%[-_0^#.:+\d]|%f', ENTITY_TIME_FORMAT) + else None) source_format = ENTITY_FORMAT - if ENTITY_FORMAT == 'csv' and ENTITY_TIME and ENTITY_TIME_FORMAT: # text, for the format below + if ENTITY_FORMAT == 'csv' and ENTITY_TIME and ARROW_FORMAT: # text for the format; '' is null source_format = ds.CsvFileFormat(convert_options=pyarrow.csv.ConvertOptions( - column_types={ENTITY_TIME: pa.string()})) + column_types={ENTITY_TIME: pa.string()}, strings_can_be_null=True)) scanner = ds.dataset(source_path, format=source_format, filesystem=source_fs).scanner(columns=list(keep)) @@ -348,12 +353,9 @@ system; the destination may be in another system (`adapter.md` §4) and plays no for f in scanner.projected_schema]) def to_column(col, field): # text in the entity's format reads as the graph reads it - if (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT + if (field.name == 'TIMESTAMP' and ARROW_FORMAT and (pa.types.is_string(col.type) or pa.types.is_large_string(col.type))): - try: - return pc.strptime(col, format=ENTITY_TIME_FORMAT, unit=field.type.unit) - except pa.ArrowInvalid: - pass # a chrono directive Arrow cannot read, e.g. %.f: cast instead + return pc.strptime(col, format=ARROW_FORMAT, unit=field.type.unit) # raises on a misfit return pc.cast(col, field.type) pop_fs, pop_dir = arrow_fs(POPULATION_PATH) @@ -381,12 +383,14 @@ system; the destination may be in another system (`adapter.md` §4) and plays no `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. `ENTITY_TIME_FORMAT` is the `timestamp_format` the entity `Table` declares on its time column (`kumoai-reference/references/graph_construction.md`, *A time column must be a real date*), else - `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. Text is parsed - with the format first, because ISO-looking text such as `2024-01-02` under `%Y-%d-%m` means what - the format says, as it does in the graph. That format is Kumo's (chrono) syntax, and Arrow's - parser rejects `%.f`, `%-m` and `%:z`; when it cannot parse, the column is cast instead, which - reads ISO-8601 text, fractional seconds included. If the cast then fails, report the format and - the failing value rather than guessing another format. The population is + `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The format is + Kumo's (chrono) syntax. When Arrow can parse it too, a CSV time column is read as text, with an + empty cell as null, and parsed with the format, because ISO-looking text such as `2024-01-02` + under `%Y-%d-%m` means what the format says, as it does in the graph. Arrow cannot parse + chrono's own directives (`%.f`, `%-m`, `%:z`); with one of those the column is read and cast as + without a format, which keeps the type the reader inferred: nanoseconds, or a UTC offset as a + time zone. If the parse or the cast fails, report the format and the failing value rather than + guessing another format. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed it, both in the config cell. Each run replaces its `population.parquet`, as Node 1d says. The diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index d2365482c..c4bef817d 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-18-83ce9a89`** +**`SKILL-BUILD: 2026-11-18-7218a192-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/hosts/generic/runtime.md b/skills/kumo-data-science/hosts/generic/runtime.md index 683034bb9..558400fc9 100644 --- a/skills/kumo-data-science/hosts/generic/runtime.md +++ b/skills/kumo-data-science/hosts/generic/runtime.md @@ -315,6 +315,7 @@ system; the destination may be in another system (`adapter.md` §4) and plays no ```python import os + import re import pyarrow as pa import pyarrow.compute as pc import pyarrow.csv @@ -334,10 +335,14 @@ system; the destination may be in another system (`adapter.md` §4) and plays no if ENTITY_TIME: # the entity Table's time_column, else None keep[ENTITY_TIME] = 'TIMESTAMP' source_fs, source_path = arrow_fs(ENTITY_SOURCE) + # Arrow parses C-style directives only; chrono's own (%.f, %-m, %:z, %3f, %f) it cannot. + ARROW_FORMAT = (ENTITY_TIME_FORMAT + if ENTITY_TIME_FORMAT and not re.search(r'%[-_0^#.:+\d]|%f', ENTITY_TIME_FORMAT) + else None) source_format = ENTITY_FORMAT - if ENTITY_FORMAT == 'csv' and ENTITY_TIME and ENTITY_TIME_FORMAT: # text, for the format below + if ENTITY_FORMAT == 'csv' and ENTITY_TIME and ARROW_FORMAT: # text for the format; '' is null source_format = ds.CsvFileFormat(convert_options=pyarrow.csv.ConvertOptions( - column_types={ENTITY_TIME: pa.string()})) + column_types={ENTITY_TIME: pa.string()}, strings_can_be_null=True)) scanner = ds.dataset(source_path, format=source_format, filesystem=source_fs).scanner(columns=list(keep)) @@ -348,12 +353,9 @@ system; the destination may be in another system (`adapter.md` §4) and plays no for f in scanner.projected_schema]) def to_column(col, field): # text in the entity's format reads as the graph reads it - if (field.name == 'TIMESTAMP' and ENTITY_TIME_FORMAT + if (field.name == 'TIMESTAMP' and ARROW_FORMAT and (pa.types.is_string(col.type) or pa.types.is_large_string(col.type))): - try: - return pc.strptime(col, format=ENTITY_TIME_FORMAT, unit=field.type.unit) - except pa.ArrowInvalid: - pass # a chrono directive Arrow cannot read, e.g. %.f: cast instead + return pc.strptime(col, format=ARROW_FORMAT, unit=field.type.unit) # raises on a misfit return pc.cast(col, field.type) pop_fs, pop_dir = arrow_fs(POPULATION_PATH) @@ -381,12 +383,14 @@ system; the destination may be in another system (`adapter.md` §4) and plays no `ENTITY_FORMAT` its format as preflight found it, `'csv'` or `'parquet'`. `ENTITY_TIME_FORMAT` is the `timestamp_format` the entity `Table` declares on its time column (`kumoai-reference/references/graph_construction.md`, *A time column must be a real date*), else - `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. Text is parsed - with the format first, because ISO-looking text such as `2024-01-02` under `%Y-%d-%m` means what - the format says, as it does in the graph. That format is Kumo's (chrono) syntax, and Arrow's - parser rejects `%.f`, `%-m` and `%:z`; when it cannot parse, the column is cast instead, which - reads ISO-8601 text, fractional seconds included. If the cast then fails, report the format and - the failing value rather than guessing another format. The population is + `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The format is + Kumo's (chrono) syntax. When Arrow can parse it too, a CSV time column is read as text, with an + empty cell as null, and parsed with the format, because ISO-looking text such as `2024-01-02` + under `%Y-%d-%m` means what the format says, as it does in the graph. Arrow cannot parse + chrono's own directives (`%.f`, `%-m`, `%:z`); with one of those the column is read and cast as + without a format, which keeps the type the reader inferred: nanoseconds, or a UTC offset as a + time zone. If the parse or the cast fails, report the format and the failing value rather than + guessing another format. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed it, both in the config cell. Each run replaces its `population.parquet`, as Node 1d says. The From bef5ab4ef46799a880012ea0327c4a0ef00fa150 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Thu, 19 Nov 2026 00:37:35 +0000 Subject: [PATCH 33/35] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/kumo-custom-populations/BENCHMARK.md | 38 +++++----- skills/kumo-custom-populations/skill-card.md | 26 +++---- skills/kumo-custom-populations/skill.oms.sig | 2 +- skills/kumo-data-science/BENCHMARK.md | 64 ++++++++-------- skills/kumo-data-science/skill-card.md | 20 ++--- skills/kumo-data-science/skill.oms.sig | 2 +- skills/kumo-graph-visualization/BENCHMARK.md | 40 +++++----- skills/kumo-graph-visualization/skill-card.md | 45 +++++------ skills/kumo-graph-visualization/skill.oms.sig | 2 +- skills/kumo-model-explanations/BENCHMARK.md | 44 +++++------ skills/kumo-model-explanations/skill-card.md | 44 +++++------ skills/kumo-model-explanations/skill.oms.sig | 2 +- skills/kumo-model-persistence/BENCHMARK.md | 46 +++++------ skills/kumo-model-persistence/skill-card.md | 76 ++++++++----------- skills/kumo-model-persistence/skill.oms.sig | 2 +- skills/kumo-model-tuning/BENCHMARK.md | 44 +++++------ skills/kumo-model-tuning/skill-card.md | 34 ++++----- skills/kumo-model-tuning/skill.oms.sig | 2 +- skills/kumo-training-diagnostics/BENCHMARK.md | 44 +++++------ .../kumo-training-diagnostics/skill-card.md | 37 +++++---- .../kumo-training-diagnostics/skill.oms.sig | 2 +- skills/kumoai-reference/BENCHMARK.md | 50 ++++++------ skills/kumoai-reference/skill-card.md | 65 ++++++++-------- skills/kumoai-reference/skill.oms.sig | 2 +- 24 files changed, 357 insertions(+), 376 deletions(-) diff --git a/skills/kumo-custom-populations/BENCHMARK.md b/skills/kumo-custom-populations/BENCHMARK.md index d6301d187..edfa72e57 100644 --- a/skills/kumo-custom-populations/BENCHMARK.md +++ b/skills/kumo-custom-populations/BENCHMARK.md @@ -50,11 +50,11 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 95.1% — baseline ran, but no comparable score was available; uplift unavailable | 90.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 97.3% — baseline ran, but no comparable score was available; uplift unavailable | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 48.0% → 100.0% (+52.0 points) | 64.0% → 92.0% (+28.0 points) | -| Discoverability | 97.5% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 48.0% → 84.0% (+36.0 points) | 44.0% → 68.0% (+24.0 points) | +| Correctness | 52.0% → 100.0% (+48.0 points) | 64.0% → 96.0% (+32.0 points) | +| Discoverability | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 44.0% → 96.0% (+52.0 points) | 42.0% → 84.0% (+42.0 points) | | Efficiency | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.0% → 96.2% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 96.4% (-1.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.4% → 92.1% (+2.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 88.4% → 92.2% (+3.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 612,687 | 579,473 | +33,214 | +5.73% | skill 5/5; base 5/5 | -| claude-code | kpop-01 | 154,399 | 110,856 | +43,543 | +39.28% | skill 1/1; base 1/1 | -| claude-code | kpop-02 | 153,205 | 143,636 | +9,569 | +6.66% | skill 1/1; base 1/1 | -| claude-code | kpop-03 | 155,016 | 107,009 | +48,007 | +44.86% | skill 1/1; base 1/1 | -| claude-code | kpop-04 | 114,314 | 182,168 | -67,854 | -37.25% | skill 1/1; base 1/1 | -| claude-code | kpop-05 | 35,753 | 35,804 | -51 | -0.14% | skill 1/1; base 1/1 | -| codex | All cases | 193,397 | 256,704 | -63,307 | -24.66% | skill 5/5; base 5/5 | -| codex | kpop-01 | 43,779 | 57,412 | -13,633 | -23.75% | skill 1/1; base 1/1 | -| codex | kpop-02 | 43,773 | 56,327 | -12,554 | -22.29% | skill 1/1; base 1/1 | -| codex | kpop-03 | 43,992 | 41,800 | +2,192 | +5.24% | skill 1/1; base 1/1 | -| codex | kpop-04 | 44,154 | 83,522 | -39,368 | -47.13% | skill 1/1; base 1/1 | -| codex | kpop-05 | 17,699 | 17,643 | +56 | +0.32% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 806,084 | 836,177 | -30,093 | -3.60% | skill 10/10; base 10/10 | +| claude-code | All cases | 610,417 | 616,054 | -5,637 | -0.92% | skill 5/5; base 5/5 | +| claude-code | kpop-01 | 154,134 | 145,524 | +8,610 | +5.92% | skill 1/1; base 1/1 | +| claude-code | kpop-02 | 153,390 | 182,068 | -28,678 | -15.75% | skill 1/1; base 1/1 | +| claude-code | kpop-03 | 113,190 | 107,162 | +6,028 | +5.63% | skill 1/1; base 1/1 | +| claude-code | kpop-04 | 154,047 | 145,611 | +8,436 | +5.79% | skill 1/1; base 1/1 | +| claude-code | kpop-05 | 35,656 | 35,689 | -33 | -0.09% | skill 1/1; base 1/1 | +| codex | All cases | 209,499 | 284,139 | -74,640 | -26.27% | skill 5/5; base 5/5 | +| codex | kpop-01 | 60,262 | 66,089 | -5,827 | -8.82% | skill 1/1; base 1/1 | +| codex | kpop-02 | 43,725 | 56,299 | -12,574 | -22.33% | skill 1/1; base 1/1 | +| codex | kpop-03 | 43,942 | 60,209 | -16,267 | -27.02% | skill 1/1; base 1/1 | +| codex | kpop-04 | 43,867 | 83,937 | -40,070 | -47.74% | skill 1/1; base 1/1 | +| codex | kpop-05 | 17,703 | 17,605 | +98 | +0.56% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 819,916 | 900,193 | -80,277 | -8.92% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-custom-populations/skill-card.md b/skills/kumo-custom-populations/skill-card.md index eee914a9e..ca038fe3d 100644 --- a/skills/kumo-custom-populations/skill-card.md +++ b/skills/kumo-custom-populations/skill-card.md @@ -30,10 +30,10 @@ Mitigation: Review and scan skill before deployment.
## Skill Output:
-**Output Type(s):** [Analysis, Code]
+**Output Type(s):** [Analysis, Code, Configuration instructions]
**Output Format:** [Markdown with inline Python code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Presents a plan and stops for user confirmation before generating training or scoring code.]
+**Other Properties Related to Output:** [Presents a plan and stops for user confirmation before generating code]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -42,7 +42,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`); evaluator version `1.6.10`; evaluated 2026-11-18.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod; each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -74,11 +74,11 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 95.1% — baseline ran, but no comparable score was available; uplift unavailable | 90.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 97.3% — baseline ran, but no comparable score was available; uplift unavailable | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 48.0% → 100.0% (+52.0 points) | 64.0% → 92.0% (+28.0 points) | -| Discoverability | 97.5% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 48.0% → 84.0% (+36.0 points) | 44.0% → 68.0% (+24.0 points) | +| Correctness | 52.0% → 100.0% (+48.0 points) | 64.0% → 96.0% (+32.0 points) | +| Discoverability | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 44.0% → 96.0% (+52.0 points) | 42.0% → 84.0% (+42.0 points) | | Efficiency | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -94,17 +94,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.0% → 96.2% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 96.4% (-1.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.4% → 92.1% (+2.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | - -### Tier Status - -| Tier | Purpose | Status | Evidence | -|---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 11 finding(s) | -| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | +| codex | Efficiency | Token-only comparison | 88.4% → 92.2% (+3.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
0.1.0 (source: pyproject.toml)
diff --git a/skills/kumo-custom-populations/skill.oms.sig b/skills/kumo-custom-populations/skill.oms.sig index 013d21387..4da83496c 100644 --- a/skills/kumo-custom-populations/skill.oms.sig +++ b/skills/kumo-custom-populations/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1jdXN0b20tcG9wdWxhdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMjJiMTNjMGUzN2MxYWYxZDM1MzQ4MTVmOGI3OGMxOGIxN2M4NmIxZjZjODdhYzE2ZGNiYmJkNDAwNDAwOTJlMCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjAyMmViYTQ4NGFhNTMzNWFhYzE4ODc3OTk2MWZlMDhjMTVhNTU5NWYyNmI2NzgxMzEyNDNlNDFkNzk3M2I4MTIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjIyODJmZDM2NDJiNDMzMzY2NTYyYmE1OWE1M2JhNWI5MzAyNTdlNzNlM2FkNThiOTZkNjhjMjcwMGYzOTc1OCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjcwNWY1NmMwZDcyZjM1YmVhOWQyMTMwZTU2MmU1MGY5OWExM2I5MTE2ZDlmOWQ0ZDNkN2U1Zjk3ZGFjOWQ3ZDgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BvcHVsYXRpb25zLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3ZjIyZmI0MzdlMGNmMDBkMWMwY2VlZTYyYTBiYjdlZjBhZjM1NDhkMzQyMjllZmZkMWMwNTVjYjllMmFkMjdhIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNTIzMDU4NTJhOTE1ODk5NDI0NTUxYTY4ZDAyODQ4ZTU2ZGYxMzZkM2QyMjM0ZjNhNDE3YzFmZDJhYTNmOGJjZSIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQD/TX010LpYPu2x2UliUA2YcUsLXc3lO/dU3RJPLzoxFQQrKc1BNJa3ARUViOe6/OECMB6g8LSywJK3JynkVBFiQe8PKd4ZNLhcmOzIilV/s37ulSf3XKOp1NUnEE/1Gmx10g==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1jdXN0b20tcG9wdWxhdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYWY1MmIyOWI1NjIzMzM1YWY2NDNlNWM0MzI0ZDA3NTAxMTFkOWEzYjI1NDZhOTY1ZmUwOWJlYjVkNmExZjAzYyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIwMTQ0MGIyOGNiZDA5YzE1N2RiNjgwODgyNzJlMTFmN2ZhYzkwZjc3ZTQ4NTcxYzFkNDU5NTEwNjI4ZWY0YjY1IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjYyMjgyZmQzNjQyYjQzMzM2NjU2MmJhNTlhNTNiYTViOTMwMjU3ZTczZTNhZDU4Yjk2ZDY4YzI3MDBmMzk3NTgiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICI3MDVmNTZjMGQ3MmYzNWJlYTlkMjEzMGU1NjJlNTBmOTlhMTNiOTExNmQ5ZjlkNGQzZDdlNWY5N2RhYzlkN2Q4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcG9wdWxhdGlvbnMubWQiLAogICAgICAgICJkaWdlc3QiOiAiN2YyMmZiNDM3ZTBjZjAwZDFjMGNlZWU2MmEwYmI3ZWYwYWYzNTQ4ZDM0MjI5ZWZmZDFjMDU1Y2I5ZTJhZDI3YSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjUyYmZjNWZjMzJhZmViZjM2ZTdmMjI1ODBiNmVlZDg1MDA0OTI3YTZmYTlhNTE3ZjY1Y2ViNGI1YWI0MzM0MmQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMET9OMbkC3KjVYlwQ+HUyK+XVTp8Qo6Y8g8VvlXhajtcufb1yQpuX2D1bz7oFlAyiwIxALGZe9eIF0zTTB41++Bj3QpLp1w1iVa6FzQ5OzMzRGhDfEiJtF7TwOYk5rvqk2mg9g==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-data-science/BENCHMARK.md b/skills/kumo-data-science/BENCHMARK.md index b60d051e5..48d198777 100644 --- a/skills/kumo-data-science/BENCHMARK.md +++ b/skills/kumo-data-science/BENCHMARK.md @@ -10,7 +10,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-data-science` - Validation status: `passed` -- Evaluation date: 2026-11-18 +- Evaluation date: 2026-11-19 - Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 8 evaluation tasks (7 positive, 1 negative) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 98.4% — baseline ran, but no comparable score was available; uplift unavailable | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 57.5% → 100.0% (+42.5 points) | 50.0% → 97.5% (+47.5 points) | -| Discoverability | 99.3% — baseline ran, but no comparable score was available; uplift unavailable | 92.9% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 41.4% → 97.9% (+56.5 points) | 36.2% → 93.8% (+57.6 points) | -| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 97.6% — baseline ran, but no comparable score was available; uplift unavailable | 95.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 93.8% → 100.0% (+6.2 points) | +| Correctness | 47.5% → 100.0% (+52.5 points) | 45.0% → 100.0% (+55.0 points) | +| Discoverability | 98.6% — baseline ran, but no comparable score was available; uplift unavailable | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 34.7% → 93.8% (+59.1 points) | 36.0% → 92.2% (+56.2 points) | +| Efficiency | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | 95.3% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.0% → 92.6% (-4.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 96.9% → 91.8% (-5.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 80.4% → 90.3% (+9.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 79.9% → 90.5% (+10.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,25 +80,25 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,456,211 | 933,207 | +523,004 | +56.04% | skill 8/8; base 8/8 | -| claude-code | kds-01 | 225,387 | 107,911 | +117,476 | +108.86% | skill 1/1; base 1/1 | -| claude-code | kds-02 | 183,714 | 201,375 | -17,661 | -8.77% | skill 1/1; base 1/1 | -| claude-code | kds-03 | 229,800 | 158,004 | +71,796 | +45.44% | skill 1/1; base 1/1 | -| claude-code | kds-04 | 222,519 | 108,540 | +113,979 | +105.01% | skill 1/1; base 1/1 | -| claude-code | kds-05 | 217,354 | 107,133 | +110,221 | +102.88% | skill 1/1; base 1/1 | -| claude-code | kds-06 | 169,879 | 107,398 | +62,481 | +58.18% | skill 1/1; base 1/1 | -| claude-code | kds-07 | 171,634 | 107,252 | +64,382 | +60.03% | skill 1/1; base 1/1 | -| claude-code | kds-08 | 35,924 | 35,594 | +330 | +0.93% | skill 1/1; base 1/1 | -| codex | All cases | 601,311 | 2,529,905 | -1,928,594 | -76.23% | skill 8/8; base 8/8 | -| codex | kds-01 | 70,062 | 45,543 | +24,519 | +53.84% | skill 1/1; base 1/1 | -| codex | kds-02 | 96,604 | 1,481,282 | -1,384,678 | -93.48% | skill 1/1; base 1/1 | -| codex | kds-03 | 122,113 | 819,648 | -697,535 | -85.10% | skill 1/1; base 1/1 | -| codex | kds-04 | 94,265 | 41,384 | +52,881 | +127.78% | skill 1/1; base 1/1 | -| codex | kds-05 | 71,914 | 30,254 | +41,660 | +137.70% | skill 1/1; base 1/1 | -| codex | kds-06 | 83,337 | 41,036 | +42,301 | +103.08% | skill 1/1; base 1/1 | -| codex | kds-07 | 49,343 | 57,280 | -7,937 | -13.86% | skill 1/1; base 1/1 | -| codex | kds-08 | 13,673 | 13,478 | +195 | +1.45% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 2,057,522 | 3,463,112 | -1,405,590 | -40.59% | skill 16/16; base 16/16 | +| claude-code | All cases | 1,532,548 | 1,046,711 | +485,837 | +46.42% | skill 8/8; base 8/8 | +| claude-code | kds-01 | 179,990 | 142,821 | +37,169 | +26.02% | skill 1/1; base 1/1 | +| claude-code | kds-02 | 242,679 | 161,366 | +81,313 | +50.39% | skill 1/1; base 1/1 | +| claude-code | kds-03 | 236,518 | 276,359 | -39,841 | -14.42% | skill 1/1; base 1/1 | +| claude-code | kds-04 | 267,189 | 144,990 | +122,199 | +84.28% | skill 1/1; base 1/1 | +| claude-code | kds-05 | 170,568 | 70,704 | +99,864 | +141.24% | skill 1/1; base 1/1 | +| claude-code | kds-06 | 217,878 | 107,637 | +110,241 | +102.42% | skill 1/1; base 1/1 | +| claude-code | kds-07 | 181,830 | 107,195 | +74,635 | +69.63% | skill 1/1; base 1/1 | +| claude-code | kds-08 | 35,896 | 35,639 | +257 | +0.72% | skill 1/1; base 1/1 | +| codex | All cases | 572,619 | 2,940,672 | -2,368,053 | -80.53% | skill 8/8; base 8/8 | +| codex | kds-01 | 69,610 | 49,453 | +20,157 | +40.76% | skill 1/1; base 1/1 | +| codex | kds-02 | 97,805 | 1,448,814 | -1,351,009 | -93.25% | skill 1/1; base 1/1 | +| codex | kds-03 | 96,860 | 1,286,574 | -1,189,714 | -92.47% | skill 1/1; base 1/1 | +| codex | kds-04 | 93,764 | 13,570 | +80,194 | +590.97% | skill 1/1; base 1/1 | +| codex | kds-05 | 84,347 | 30,117 | +54,230 | +180.06% | skill 1/1; base 1/1 | +| codex | kds-06 | 66,995 | 41,092 | +25,903 | +63.04% | skill 1/1; base 1/1 | +| codex | kds-07 | 49,516 | 57,606 | -8,090 | -14.04% | skill 1/1; base 1/1 | +| codex | kds-08 | 13,722 | 13,446 | +276 | +2.05% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,105,167 | 3,987,383 | -1,882,216 | -47.20% | skill 16/16; base 16/16 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -118,6 +118,10 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - **HIGH** DUPLICATE/duplicate: Duplicate content found across SKILL.md and accept/preflight.md: "### Node 1c — Preflight & graph gate" in SKILL.md (lines 178-194) vs "## 5. Column availability" in accept/preflight.md (lines 157-171) (`SKILL.md:178`) +- **HIGH** DUPLICATE/duplicate: Duplicate content found across accept/intake_contract.md and validate/examples/good_request.md: + "## Canonical good example (`train`)" in accept/intake_contract.md (lines 137-148) + vs "## Input" in validate/examples/good_request.md (lines 3-12) + vs "## Node 1a — Required fields: PASS" in validate/examples/good_request.md (lines 13-19) (`accept/intake_contract.md:137`) - **HIGH** DUPLICATE/duplicate: Duplicate content found across validate/atomic_question_check.md and validate/examples/bad_request.md: "### Fail 1 — Multiple targets bundled" in validate/atomic_question_check.md (lines 42-50) vs "## 6. Bundled goals → REJECT" in validate/examples/bad_request.md (lines 64-68) (`validate/atomic_question_check.md:42`) @@ -125,10 +129,6 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c "## Node 1a — fields" in accept/batch_inference.md (lines 7-29) vs "### `batch_inference` — required fields (ask if missing)" in accept/batch_inference.md (lines 30-42) vs "### `batch_inference` — resolved, defaulted, or inferred (never ask)" in accept/batch_inference.md (lines 43-54) (`accept/batch_inference.md:7`) -- **HIGH** DUPLICATE/duplicate: Duplicate content found across accept/intake_contract.md and validate/examples/good_request.md: - "## Canonical good example (`train`)" in accept/intake_contract.md (lines 137-148) - vs "## Input" in validate/examples/good_request.md (lines 3-12) - vs "## Node 1a — Required fields: PASS" in validate/examples/good_request.md (lines 13-19) (`accept/intake_contract.md:137`) - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-data-science/SKILL.md`) - 25 additional finding(s) are available in the full evaluation artifacts. diff --git a/skills/kumo-data-science/skill-card.md b/skills/kumo-data-science/skill-card.md index 20b3d149a..4f25af972 100644 --- a/skills/kumo-data-science/skill-card.md +++ b/skills/kumo-data-science/skill-card.md @@ -50,7 +50,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-8 evaluation tasks (7 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox pod (`k8s-sandbox`), compared against a no-skill baseline. Evaluator version `1.6.10`; evaluated 2026-11-18.
+8 evaluation tasks (7 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox pod (`k8s-sandbox`), compared against a no-skill baseline. Evaluator version `1.6.10`; evaluated 2026-11-19.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -76,12 +76,12 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 98.4% — baseline ran, but no comparable score was available; uplift unavailable | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 57.5% → 100.0% (+42.5 points) | 50.0% → 97.5% (+47.5 points) | -| Discoverability | 99.3% — baseline ran, but no comparable score was available; uplift unavailable | 92.9% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 41.4% → 97.9% (+56.5 points) | 36.2% → 93.8% (+57.6 points) | -| Efficiency | 94.5% — baseline ran, but no comparable score was available; uplift unavailable | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 97.6% — baseline ran, but no comparable score was available; uplift unavailable | 95.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 93.8% → 100.0% (+6.2 points) | +| Correctness | 47.5% → 100.0% (+52.5 points) | 45.0% → 100.0% (+55.0 points) | +| Discoverability | 98.6% — baseline ran, but no comparable score was available; uplift unavailable | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 34.7% → 93.8% (+59.1 points) | 36.0% → 92.2% (+56.2 points) | +| Efficiency | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | 95.3% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -96,9 +96,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.0% → 92.6% (-4.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 96.9% → 91.8% (-5.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 80.4% → 90.3% (+9.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 79.9% → 90.5% (+10.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -113,7 +113,7 @@ Shared-source comparisons use the same signals and weights on both conditions. T **Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. ## Skill Version(s):
-0.1.0 (source: pyproject.toml); skill build marker 2026-11-18-83ce9a89 (source: SKILL.md); git 8aaa6cb3 (committed 2026-11-18)
+0.1.0 (source: pyproject.toml); skill build marker 2026-11-18-7218a192-dirty (source: SKILL.md); git bfbf737f (committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-data-science/skill.oms.sig b/skills/kumo-data-science/skill.oms.sig index 192dd7d34..7bbb17bc0 100644 --- a/skills/kumo-data-science/skill.oms.sig +++ b/skills/kumo-data-science/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1kYXRhLXNjaWVuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiOWNiYjliOTAwNDRlM2EyYzQ5ZTc3ZTJjNDUyOWYxZGY3YzI1YTcxMmY3ZmI4OGFhMzdmN2NmZTc1MDhiMjI5ZSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZTA4ZDI3MjJkZTExZDc1NzVlYzI4ODNmMzBhNmZhNjAxNGZlZTJmMmFkODE5MTNiOTRmMjdiZjIyMWZmODRkYSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3ZDUwZTA0YWE4YTU1ODFiMjVhNDY4YTYxZmQ5YjM1MTJkNDM4MDRiZjc0ZDczNWYxMjA0NzRkMzhkYWM3Y2U2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L2JhdGNoX2luZmVyZW5jZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMmQ3NWFiYTBmYTRmZTE5Yjg4NTY1MzI4MGUxYTUyZDEzYjA3YTBlMTMzZGM0MzRkMWIyYmQ3YWIwZjlmNmJiOCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImFjY2VwdC9pbnRha2VfY29udHJhY3QubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImZlYTdiYmQ4NjViODM2NjQxMTA3Zjc0MDM0ODI2YjFmNTRhMTRkMTRmMzhiMTdlYWMwZmRkYTgwODk0MTIyZDEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJhY2NlcHQvcHJlZmxpZ2h0Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkM2I0YzkwNzQxNWVhNDM2MGUwOWE0YWEyMmU5Y2E4ZTY4YmQyM2Y4YjRjNDdjY2Y5ZDFiOTVkOGFhNjFiZTlmIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L3NlbWFudGljX3ZpZXcubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImY5ZjJkZjljYWE0ZDU2OWU0M2U0ZDM0ZjcwOTliMjE1MTFmYWMxYWIzOWVkNjIyNzMyNDhmYjYxMWI5Y2JkM2UiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2NjQwNGQ5Y2IxM2QwMDkzYjY3MDFkZmQ0YjE5MzM3MTAzZDhkM2I4NjA3NWUxMTEzOTM2ZDNiNDRlOTljY2E5IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiaG9zdHMvZ2VuZXJpYy9hZGFwdGVyLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJjNmEzNjg0NTRmYTlhZGVlYmEwYmRkYTk4YmQzZDIxNGYyYjkyYTVmMTI4MzUyNzQ1YzNkMjJlYjkwN2I1YjZiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiaG9zdHMvZ2VuZXJpYy9ydW50aW1lLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlMjM5ZTJkODQ2NWFiYWI3OWMyN2YyNmIyMWViY2M3Nzc0MzM5YjQxMDNhMDM2MGY4MjJmNzhhZmMyNGNlOTBiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzE5NTRiMzFiODAzNjg4ZmViMTUxNTZiNzBmMGQxYmQzNGQ5ZmZjYzRlZTIyMmM3OGVjMWM2Y2Y5MjE1NWRiMiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2F0b21pY19xdWVzdGlvbl9jaGVjay5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjUwNmU2NjFiYzExYzI1ZWI2NmE3YjdjMzVhMzQwMzEzOTU3ZDMyNGJhMzg2N2U5NTA4NmIxMTRlOTJkMjA0YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2V4YW1wbGVzL2JhZF9yZXF1ZXN0Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3YmQzOTM0ZTFiOWMyMmE0YzM4YjM3YmNiOGM1OTcyOTJkZGVlODIyMDYxZDlmZWVlZTY1N2JkOTA1MDM1Y2Q0IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAidmFsaWRhdGUvZXhhbXBsZXMvZ29vZF9yZXF1ZXN0Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI4OTJkMGZiZWI0MmE4MDJkMzM2ZmZiMjMwOWQ1ZTk5OWIzOGNmYjQ4N2QzNjljMzg5Mjk0OWYzY2MxYmVkOGU2IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDZezCWX0j2hma/i4Yfp6dahzVDS7Z5ogIIv9lI0ulzp75pPNQsf2u+5JQMRtv7JEcCMFFm5UW8A9K1p+7TMzF//1Jbutj7HFhyg271tZNG7V08gs89zX9e+33LiXPr3DbltQ==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1kYXRhLXNjaWVuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiOWM1ZDllNWMyNjY1MjhiMDYyMzBlZTkwY2MyZjNlNWM3MmQ5NzkxZGY2YzU5NzRlNzVmNmRjMjE1NGFmYjcyYiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQwNjg3NGJlMDc1MmQzZmFmMTA5YzQ5YWMwNmFhYjdjOWM3Njc4YTM3MzJhYmIzZWFhYTI3OGY1OTc4NDY5NWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiM2E2Y2RlZDQyZTQ0YzRkNTliMDhkMDNiNTMzZTg3NjFmNzhlNGMyZDhlNWMzMTU2Nzc3MDdkYjVhMWEyMzVhYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImFjY2VwdC9iYXRjaF9pbmZlcmVuY2UubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjJkNzVhYmEwZmE0ZmUxOWI4ODU2NTMyODBlMWE1MmQxM2IwN2EwZTEzM2RjNDM0ZDFiMmJkN2FiMGY5ZjZiYjgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJhY2NlcHQvaW50YWtlX2NvbnRyYWN0Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmZWE3YmJkODY1YjgzNjY0MTEwN2Y3NDAzNDgyNmIxZjU0YTE0ZDE0ZjM4YjE3ZWFjMGZkZGE4MDg5NDEyMmQxIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L3ByZWZsaWdodC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDNiNGM5MDc0MTVlYTQzNjBlMDlhNGFhMjJlOWNhOGU2OGJkMjNmOGI0YzQ3Y2NmOWQxYjk1ZDhhYTYxYmU5ZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImFjY2VwdC9zZW1hbnRpY192aWV3Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmOWYyZGY5Y2FhNGQ1NjllNDNlNGQzNGY3MDk5YjIxNTExZmFjMWFiMzllZDYyMjczMjQ4ZmI2MTFiOWNiZDNlIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjY0MDRkOWNiMTNkMDA5M2I2NzAxZGZkNGIxOTMzNzEwM2Q4ZDNiODYwNzVlMTExMzkzNmQzYjQ0ZTk5Y2NhOSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImhvc3RzL2dlbmVyaWMvYWRhcHRlci5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYzZhMzY4NDU0ZmE5YWRlZWJhMGJkZGE5OGJkM2QyMTRmMmI5MmE1ZjEyODM1Mjc0NWMzZDIyZWI5MDdiNWI2YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImhvc3RzL2dlbmVyaWMvcnVudGltZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjZjNmM5NDFiMWQwZTdlYTA5OWJjNTRjNzBmODlkZWYwZGM4NDI5YmNjZDQzYjY1MGQ5NmMyNThjYzhmZWNmYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjU3YTEyMWU2NjFiNDlhYTIzYmRjMWI2ZjcwNTVkMzVlNmQ1NmExYzBjOGNjMzQ4MTdkMTc5NTIwNjBlMzQyMzgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJ2YWxpZGF0ZS9hdG9taWNfcXVlc3Rpb25fY2hlY2subWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjY1MDZlNjYxYmMxMWMyNWViNjZhN2I3YzM1YTM0MDMxMzk1N2QzMjRiYTM4NjdlOTUwODZiMTE0ZTkyZDIwNGEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJ2YWxpZGF0ZS9leGFtcGxlcy9iYWRfcmVxdWVzdC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2JkMzkzNGUxYjljMjJhNGMzOGIzN2JjYjhjNTk3MjkyZGRlZTgyMjA2MWQ5ZmVlZWU2NTdiZDkwNTAzNWNkNCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2V4YW1wbGVzL2dvb2RfcmVxdWVzdC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiODkyZDBmYmViNDJhODAyZDMzNmZmYjIzMDlkNWU5OTliMzhjZmI0ODdkMzY5YzM4OTI5NDlmM2NjMWJlZDhlNiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDa7oeoHp4Wc28ILUajCblNJ7/7/2P51d+mllAZd/qWDZ8OZ3at5+VJnNTbPuU1IXgCMEFOFWSlEepXDqXzpePUSFNAeQFQuZ33WLbI0wmDebMbP55QtXZvGkyOr8h45H08LQ==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-graph-visualization/BENCHMARK.md b/skills/kumo-graph-visualization/BENCHMARK.md index d58f8c9a7..194682097 100644 --- a/skills/kumo-graph-visualization/BENCHMARK.md +++ b/skills/kumo-graph-visualization/BENCHMARK.md @@ -10,7 +10,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-graph-visualization` - Validation status: `passed` -- Evaluation date: 2026-11-18 +- Evaluation date: 2026-11-19 - Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 5 evaluation tasks (4 positive, 1 negative) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.2% — baseline ran, but no comparable score was available; uplift unavailable | 79.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | 78.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | | Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 72.0% (+44.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 30.8% → 60.0% (+29.2 points) | 30.0% → 37.5% (+7.5 points) | -| Efficiency | 81.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 30.8% → 50.0% (+19.2 points) | 32.5% → 37.5% (+5.0 points) | +| Efficiency | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 97.6% (-0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 97.3% (-1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.6% → 93.0% (+1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 92.3% → 92.0% (-0.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 842,402 | 751,532 | +90,870 | +12.09% | skill 5/5; base 5/5 | -| claude-code | kviz-01 | 145,858 | 106,564 | +39,294 | +36.87% | skill 1/1; base 1/1 | -| claude-code | kviz-02 | 148,209 | 107,632 | +40,577 | +37.70% | skill 1/1; base 1/1 | -| claude-code | kviz-03 | 185,661 | 180,056 | +5,605 | +3.11% | skill 1/1; base 1/1 | -| claude-code | kviz-04 | 109,587 | 143,724 | -34,137 | -23.75% | skill 1/1; base 1/1 | -| claude-code | kviz-05 | 253,087 | 213,556 | +39,531 | +18.51% | skill 1/1; base 1/1 | -| codex | All cases | 271,299 | 225,930 | +45,369 | +20.08% | skill 5/5; base 5/5 | -| codex | kviz-01 | 57,292 | 54,986 | +2,306 | +4.19% | skill 1/1; base 1/1 | -| codex | kviz-02 | 27,459 | 46,274 | -18,815 | -40.66% | skill 1/1; base 1/1 | -| codex | kviz-03 | 72,377 | 26,738 | +45,639 | +170.69% | skill 1/1; base 1/1 | -| codex | kviz-04 | 56,957 | 57,507 | -550 | -0.96% | skill 1/1; base 1/1 | -| codex | kviz-05 | 57,214 | 40,425 | +16,789 | +41.53% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,113,701 | 977,462 | +136,239 | +13.94% | skill 10/10; base 10/10 | +| claude-code | All cases | 880,796 | 786,371 | +94,425 | +12.01% | skill 5/5; base 5/5 | +| claude-code | kviz-01 | 145,931 | 141,553 | +4,378 | +3.09% | skill 1/1; base 1/1 | +| claude-code | kviz-02 | 110,065 | 107,381 | +2,684 | +2.50% | skill 1/1; base 1/1 | +| claude-code | kviz-03 | 147,135 | 143,223 | +3,912 | +2.73% | skill 1/1; base 1/1 | +| claude-code | kviz-04 | 147,314 | 142,938 | +4,376 | +3.06% | skill 1/1; base 1/1 | +| claude-code | kviz-05 | 330,351 | 251,276 | +79,075 | +31.47% | skill 1/1; base 1/1 | +| codex | All cases | 239,429 | 244,628 | -5,199 | -2.13% | skill 5/5; base 5/5 | +| codex | kviz-01 | 57,428 | 68,472 | -11,044 | -16.13% | skill 1/1; base 1/1 | +| codex | kviz-02 | 27,482 | 41,450 | -13,968 | -33.70% | skill 1/1; base 1/1 | +| codex | kviz-03 | 57,177 | 40,594 | +16,583 | +40.85% | skill 1/1; base 1/1 | +| codex | kviz-04 | 54,891 | 53,225 | +1,666 | +3.13% | skill 1/1; base 1/1 | +| codex | kviz-05 | 42,451 | 40,887 | +1,564 | +3.83% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,120,225 | 1,030,999 | +89,226 | +8.65% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-graph-visualization/skill-card.md b/skills/kumo-graph-visualization/skill-card.md index a597601a9..2410ffe8c 100644 --- a/skills/kumo-graph-visualization/skill-card.md +++ b/skills/kumo-graph-visualization/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists using Kumo notebooks or SDK scripts use this skill to display or export graph diagrams of tables and relationships (SVG, PDF, or Markdown/Mermaid) and to inspect graph schemas while iterating on models.
+Developers and data scientists building Kumo models use this skill to add graph previews to generated notebooks and SDK scripts, display or export a graph's tables and relationships as SVG, PDF, or Markdown on request, and inspect the Markdown schema when planning graph changes.
### Deployment Geography for Use:
Global
@@ -25,16 +25,15 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [Graph construction (kumoai-reference)](../kumoai-reference/references/graph_construction.md)
-- [Kumo host runtimes (kumo-data-science)](../kumo-data-science/hosts/)
- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
+- [kumo-engine on PyPI](https://pypi.org/project/kumo-engine/)
## Skill Output:
**Output Type(s):** [Code, Files]
-**Output Format:** [Markdown with inline Python code blocks; SVG, PDF, or Markdown graph exports]
+**Output Format:** [Markdown with inline Python code blocks; exported graph files in SVG, PDF, or Markdown]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Sample values are omitted unless the user requests them]
+**Other Properties Related to Output:** [Graph viewer supports up to 30 tables and no self-relations; sample values are off unless the user requests them]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -43,7 +42,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod (`k8s-sandbox`); evaluator version `1.6.10`, evaluated 2026-11-18. Tier 2 and Tier 3 evidence required for publication.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod (`k8s-sandbox`); evaluator version `1.6.10`, evaluated 2026-11-19. Baseline runs attempted the same tasks without the skill.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -54,18 +53,18 @@ Reported benchmark dimensions:
- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Accuracy: final-answer correctness against the reference answer.
-- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
-- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
-- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
-**Overall verdict: PASS — Recommended for publication** +> ✅ **Overall verdict: PASS — Recommended for publication** **Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. @@ -73,25 +72,25 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.2% — baseline ran, but no comparable score was available; uplift unavailable | 79.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | 78.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | | Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 72.0% (+44.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 30.8% → 60.0% (+29.2 points) | 30.0% → 37.5% (+7.5 points) | -| Efficiency | 81.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 30.8% → 50.0% (+19.2 points) | 32.5% → 37.5% (+5.0 points) | +| Efficiency | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. -#### Comparison details +### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 97.6% (-0.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 97.3% (-1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.6% → 93.0% (+1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 92.3% → 92.0% (-0.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -101,8 +100,10 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | +Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. Overall verdict: PASS only when every configured dimension passes for at least one supported agent. + ## Skill Version(s):
-0.1.0 (source: pyproject.toml; git SHA 8aaa6cb3, committed 2026-11-18)
+0.1.0 (source: pyproject.toml)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-graph-visualization/skill.oms.sig b/skills/kumo-graph-visualization/skill.oms.sig index 72ee93c08..b7e9eb0df 100644 --- a/skills/kumo-graph-visualization/skill.oms.sig +++ b/skills/kumo-graph-visualization/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1ncmFwaC12aXN1YWxpemF0aW9uIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImZjYzkxMGUxY2Q1YmUwY2FhZmI2Y2M2NDA0ZDIwNmNhZDQwZDk3M2M0ODk4NmU1N2E5NzA3NzgyZDA3MjE1NjgiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2RhYmVkZjJlMWY2MTMxMjY1ODZlYjdiNGEwOTQzODRjNTEyZjQxNmI0N2M4OWY0MWQxOTc1NjcxYTIwMjYyMSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI5NTYzYzJkNWQ0ZDMyYmFiMDI4YTRmOTZiYTE1ZTQ0NWQ3YWVjMjJmMTNmNGRiZjQ0NzY1ZWNmNTNiMGM4ZmY3IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiNzc0YTI3OGE2NjkwZmUwYmY3MTAwMWQ3NjA5NWQzNjZkMTBhODAzYzhhNjY4NjVkZGYzNGEzMmU3NWU2Yjk5YiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjM5Y2Q3Y2VhMjUzNzUxYjg3MmQ0ZDlhMDJmY2QyM2MwNjNhZDg3YjAzMGJmZGNlNjE1MWM4YTNjMDZiZmFiZjciLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGh1YiIKICAgICAgXQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQD36oDp24dEG6QXJda30i8md8JdxSq1VjKhg0VXvblj/fM1BkmIyyILc4vowoXXCywCMQCyBcSTEbEwA+8CjfHO0f+AA9HYaGv9lJGrRQWNI2ASDWn+Alz1VX9ZLBhZGh+/jso=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1ncmFwaC12aXN1YWxpemF0aW9uIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImQ0ODFkNmNkNTlkNmQ2ODU5N2I3MWRhYjgwNDQwNmUwOWNmM2NjMjhhMTdhMDk4ZjZiMjZhMjJiNjhjNjQ5YTIiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0KICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIyY2FkZWU2ZDcyMjU1MGE2Zjg0OWE1MTNhODVhN2JhMzJmYjU1ODc2NDQ5YzcxN2FmZjY5N2JhYWEyMjE5MzI3IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjk1NjNjMmQ1ZDRkMzJiYWIwMjhhNGY5NmJhMTVlNDQ1ZDdhZWMyMmYxM2Y0ZGJmNDQ3NjVlY2Y1M2IwYzhmZjciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3NzRhMjc4YTY2OTBmZTBiZjcxMDAxZDc2MDk1ZDM2NmQxMGE4MDNjOGE2Njg2NWRkZjM0YTMyZTc1ZTZiOTliIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiODA3MjcxYmY3M2VmYTE3OWNmNThiOWExNGM0NDI4ZDU3NTU2YTJhOTFmZTQ4Y2E3OTk5ZmJjZjJhYTkwMzFhMCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMBHHd1ds0NngNnG3J/24Y2GBzGkZsMTDxxqT4oh7p6HxDFGorZkSml5JbK4gRyc5MAIwGSqg5GcZF3lIV4eiIpIVgGJV0EeCZfzQQ3qfluGXQ+jCTnMx6qVARkEJwWjAgL6B","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-explanations/BENCHMARK.md b/skills/kumo-model-explanations/BENCHMARK.md index 47f847e03..0ad6aa830 100644 --- a/skills/kumo-model-explanations/BENCHMARK.md +++ b/skills/kumo-model-explanations/BENCHMARK.md @@ -10,7 +10,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-model-explanations` - Validation status: `passed` -- Evaluation date: 2026-11-18 +- Evaluation date: 2026-11-19 - Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | 88.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | 87.7% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 40.0% → 100.0% (+60.0 points) | 56.7% → 93.3% (+36.6 points) | +| Correctness | 43.3% → 100.0% (+56.7 points) | 40.0% → 93.3% (+53.3 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 27.8% → 64.6% (+36.8 points) | 50.0% → 59.7% (+9.7 points) | -| Efficiency | 97.8% — baseline ran, but no comparable score was available; uplift unavailable | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 25.0% → 64.6% (+39.6 points) | 22.9% → 56.3% (+33.4 points) | +| Efficiency | 97.9% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.2% → 95.6% (-2.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.1% → 95.7% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.8% → 91.9% (+2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.6% → 91.6% (+1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,21 +80,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 966,880 | 823,367 | +143,513 | +17.43% | skill 6/6; base 6/6 | -| claude-code | kexpl-01 | 159,703 | 142,288 | +17,415 | +12.24% | skill 1/1; base 1/1 | -| claude-code | kexpl-02 | 158,941 | 107,859 | +51,082 | +47.36% | skill 1/1; base 1/1 | -| claude-code | kexpl-03 | 116,594 | 142,990 | -26,396 | -18.46% | skill 1/1; base 1/1 | -| claude-code | kexpl-04 | 160,267 | 144,212 | +16,055 | +11.13% | skill 1/1; base 1/1 | -| claude-code | kexpl-05 | 226,120 | 178,315 | +47,805 | +26.81% | skill 1/1; base 1/1 | -| claude-code | kexpl-06 | 145,255 | 107,703 | +37,552 | +34.87% | skill 1/1; base 1/1 | -| codex | All cases | 411,790 | 298,681 | +113,109 | +37.87% | skill 6/6; base 6/6 | -| codex | kexpl-01 | 81,391 | 66,405 | +14,986 | +22.57% | skill 1/1; base 1/1 | -| codex | kexpl-02 | 81,040 | 41,166 | +39,874 | +96.86% | skill 1/1; base 1/1 | -| codex | kexpl-03 | 81,498 | 53,873 | +27,625 | +51.28% | skill 1/1; base 1/1 | -| codex | kexpl-04 | 81,678 | 54,065 | +27,613 | +51.07% | skill 1/1; base 1/1 | -| codex | kexpl-05 | 45,305 | 42,610 | +2,695 | +6.32% | skill 1/1; base 1/1 | -| codex | kexpl-06 | 40,878 | 40,562 | +316 | +0.78% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,378,670 | 1,122,048 | +256,622 | +22.87% | skill 12/12; base 12/12 | +| claude-code | All cases | 886,610 | 896,669 | -10,059 | -1.12% | skill 6/6; base 6/6 | +| claude-code | kexpl-01 | 116,441 | 142,323 | -25,882 | -18.19% | skill 1/1; base 1/1 | +| claude-code | kexpl-02 | 158,908 | 144,073 | +14,835 | +10.30% | skill 1/1; base 1/1 | +| claude-code | kexpl-03 | 116,786 | 143,330 | -26,544 | -18.52% | skill 1/1; base 1/1 | +| claude-code | kexpl-04 | 159,748 | 143,054 | +16,694 | +11.67% | skill 1/1; base 1/1 | +| claude-code | kexpl-05 | 226,319 | 178,371 | +47,948 | +26.88% | skill 1/1; base 1/1 | +| claude-code | kexpl-06 | 108,408 | 145,518 | -37,110 | -25.50% | skill 1/1; base 1/1 | +| codex | All cases | 416,838 | 286,190 | +130,648 | +45.65% | skill 6/6; base 6/6 | +| codex | kexpl-01 | 81,433 | 43,854 | +37,579 | +85.69% | skill 1/1; base 1/1 | +| codex | kexpl-02 | 83,133 | 41,340 | +41,793 | +101.10% | skill 1/1; base 1/1 | +| codex | kexpl-03 | 84,670 | 56,514 | +28,156 | +49.82% | skill 1/1; base 1/1 | +| codex | kexpl-04 | 81,171 | 62,562 | +18,609 | +29.74% | skill 1/1; base 1/1 | +| codex | kexpl-05 | 45,285 | 41,057 | +4,228 | +10.30% | skill 1/1; base 1/1 | +| codex | kexpl-06 | 41,146 | 40,863 | +283 | +0.69% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,303,448 | 1,182,859 | +120,589 | +10.19% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-explanations/skill-card.md b/skills/kumo-model-explanations/skill-card.md index 45674d2c4..95025fe2b 100644 --- a/skills/kumo-model-explanations/skill-card.md +++ b/skills/kumo-model-explanations/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Inspect and interpret gradient attribution, column analysis, and sampled explanation subgraphs from a Kumo model trained in a Python notebook, locating the correct run's explanation artifacts and identifying missing or unsupported explanations without automatically retraining.
+Inspect and interpret gradient attribution (including requests called radiant attribution), column analysis and sampled explanation subgraphs from a Kumo model trained in a Python notebook, locating the correct run's explanation artifacts, explaining feature scores and cohorts, and identifying missing or unsupported explanations without automatically retraining.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill to inspect a Kumo model's retained explanation artifacts in a notebook, explain gradient attribution scores and column-analysis cohorts, and diagnose missing or unsupported explanations without starting another fit.
+Developers and data scientists use this skill to inspect and interpret explanation artifacts (gradient attribution, column-analysis cohorts, sampled subgraphs) from a Kumo model trained in a notebook, reporting provenance, coverage, and missing evidence without starting another fit.
### Deployment Geography for Use:
Global
@@ -33,7 +33,7 @@ Mitigation: Review and scan skill before deployment.
**Output Type(s):** [Analysis, Code]
**Output Format:** [Markdown with inline Python code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [None]
+**Other Properties Related to Output:** [Reports run, task, stage, selected trainer/refit, and file paths read; distinguishes observations, hypotheses, and missing evidence.]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -42,29 +42,29 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`), compared against a no-skill baseline. Evaluator version `1.6.10`; evaluation date 2026-11-18.
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod (`k8s-sandbox`); each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
- Security: Is it safe to use? Scored from `security` (100%).
- Correctness: Is the answer correct? Scored from `accuracy` (100%).
- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
-- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
-- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` (50%) and `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? `skill_efficiency` (50%) plus `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Accuracy: final-answer correctness against the reference answer.
-- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
-- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
-- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
-**Overall verdict:** ✅ PASS — Recommended for publication +**Overall verdict: PASS — Recommended for publication** **Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. @@ -72,12 +72,12 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | 88.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | 87.7% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 40.0% → 100.0% (+60.0 points) | 56.7% → 93.3% (+36.6 points) | +| Correctness | 43.3% → 100.0% (+56.7 points) | 40.0% → 93.3% (+53.3 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 27.8% → 64.6% (+36.8 points) | 50.0% → 59.7% (+9.7 points) | -| Efficiency | 97.8% — baseline ran, but no comparable score was available; uplift unavailable | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 25.0% → 64.6% (+39.6 points) | 22.9% → 56.3% (+33.4 points) | +| Efficiency | 97.9% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -85,19 +85,19 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. -### Comparison details +#### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.2% → 95.6% (-2.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.1% → 95.7% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.8% → 91.9% (+2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.6% → 91.6% (+1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
-0.1.0 (source: pyproject.toml; git SHA 8aaa6cb3, committed 2026-11-18)
+0.1.0 (source: pyproject.toml)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-model-explanations/skill.oms.sig b/skills/kumo-model-explanations/skill.oms.sig index f97db8fa3..2b9949b10 100644 --- a/skills/kumo-model-explanations/skill.oms.sig +++ b/skills/kumo-model-explanations/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1leHBsYW5hdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMjU4YTEzMjkyZmM1MTQ1YjU5ZTRjOTE0NjMxMTUzYmUwMDViZDNlYzNmYmU2OTllY2E0MGU1MDgxMWI4OWI1MyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogImJmOTk0YWM0ODkxYzc0MzI3YmIzNGE5NTBlYjZlYjcwNmJkNDlkNjkwNWQyNzMxMGEyMzYzODliMjAyMjg1NTgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiY2I3NGI3ZGRjZGQzYWUwNmM1MmJjODUyY2Q5Yzk3OWU5NGJhNjQwYzZlY2U4MjJmNWY0OTJiYmZmOTdkYWEwYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjk2NzRhNzFhMWExYmYwNmU1ZjNhMDdlNGNjZWRlNTdkMTMzNjYzOWY0NGMyMWYwMzQzYWQ3MmE3ZTA3MmRiMTciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9hcnRpZmFjdHMubWQiLAogICAgICAgICJkaWdlc3QiOiAiNTBjYjY4ZmFiZDc0YzhlMWU5MmM0YzgzMGVlNTA5ZWRjYjY2NTRiNzBmOThjMjdjZThmZTM0Yzg5OGVhNzE3YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjY3ZWZhMmNjMWE2ZWYxMmRiNTllMGQ3YmZmYzUyNmI4MzNjMmY2YzBhMzQ4OTY4YTNiMTFiZmMzNTQzYjYzMWUiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCKJ9asaviqcQ6TdgckIfgSpPDdaZRBvgJPShK7+7ckNXz7BN/UOEUAQd8LX5nOLB0CMQDU0x14ho3fJVYptNjbWjKigo3hOv+r9AQJuul/UTLF8TDqDkBim71rSYd037LUWIE=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1leHBsYW5hdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNzllMDBjYjA4NjljNDE4ZDA5ZmU4YTM2ODY0OWU4NTk4ZGM3MGRhNjFiODI3MjMyNjQ1Mzg0YTBhYTcyMDE4OCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIKICAgICAgXSwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjYwNDVhY2JiMDdmOWUzZmQ2ODdlNjg2Nzk4MmMzYjIwNjE0MjUxZjdhOTkwOWRlNjIzZjdjNzc4NjIxODQ4MTAiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImNiNzRiN2RkY2RkM2FlMDZjNTJiYzg1MmNkOWM5NzllOTRiYTY0MGM2ZWNlODIyZjVmNDkyYmJmZjk3ZGFhMGMiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOTY3NGE3MWExYTFiZjA2ZTVmM2EwN2U0Y2NlZGU1N2QxMzM2NjM5ZjQ0YzIxZjAzNDNhZDcyYTdlMDcyZGIxNyIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjUwY2I2OGZhYmQ3NGM4ZTFlOTJjNGM4MzBlZTUwOWVkY2I2NjU0YjcwZjk4YzI3Y2U4ZmUzNGM4OThlYTcxN2EiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvYXJ0aWZhY3RzLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiY2JkMzhhZWRlOTg1MzRmN2Y4ZDhjNjY2NWE2MDYwMDllMDgzYWFmOWQyOTkxNDg1YWVlM2I4MTQzNTBjNTVmMiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDzHlEctDcF654TNEYk+yVRU3bfsfHEV33M6rRQye4MHF7AwcAjODovCuNexs42PzsCMQDl7wncFGbDf3b69nktSmqrEGt3AA7EKfgfkNSk5dhroI5LyM6+eUHQ4YHP3q4ZxsU=","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-persistence/BENCHMARK.md b/skills/kumo-model-persistence/BENCHMARK.md index a186d5e7e..7b06e0c78 100644 --- a/skills/kumo-model-persistence/BENCHMARK.md +++ b/skills/kumo-model-persistence/BENCHMARK.md @@ -10,7 +10,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-model-persistence` - Validation status: `passed` -- Evaluation date: 2026-11-18 +- Evaluation date: 2026-11-19 - Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 6 evaluation tasks (5 positive, 1 negative) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | 82.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 92.1% — baseline ran, but no comparable score was available; uplift unavailable | 80.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 91.7% → 100.0% (+8.3 points) | -| Correctness | 53.3% → 90.0% (+36.7 points) | 63.3% → 86.7% (+23.4 points) | -| Discoverability | 99.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 28.1% → 72.2% (+44.1 points) | 28.9% → 50.6% (+21.7 points) | -| Efficiency | 85.9% — baseline ran, but no comparable score was available; uplift unavailable | 82.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 56.7% → 96.7% (+40.0 points) | 40.0% → 83.3% (+43.3 points) | +| Discoverability | 98.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 32.5% → 73.9% (+41.4 points) | 27.2% → 47.2% (+20.0 points) | +| Efficiency | 92.1% — baseline ran, but no comparable score was available; uplift unavailable | 82.5% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.8% → 96.5% (-1.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.1% → 96.3% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 83.5% → 89.9% (+6.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 84.7% → 89.7% (+5.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,21 +80,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,178,061 | 689,035 | +489,026 | +70.97% | skill 6/6; base 6/6 | -| claude-code | kpersist-01 | 312,188 | 146,077 | +166,111 | +113.71% | skill 1/1; base 1/1 | -| claude-code | kpersist-02 | 228,280 | 147,904 | +80,376 | +54.34% | skill 1/1; base 1/1 | -| claude-code | kpersist-03 | 228,773 | 72,041 | +156,732 | +217.56% | skill 1/1; base 1/1 | -| claude-code | kpersist-04 | 193,734 | 108,886 | +84,848 | +77.92% | skill 1/1; base 1/1 | -| claude-code | kpersist-05 | 107,344 | 106,819 | +525 | +0.49% | skill 1/1; base 1/1 | -| claude-code | kpersist-06 | 107,742 | 107,308 | +434 | +0.40% | skill 1/1; base 1/1 | -| codex | All cases | 357,551 | 640,372 | -282,821 | -44.17% | skill 6/6; base 6/6 | -| codex | kpersist-01 | 86,040 | 386,406 | -300,366 | -77.73% | skill 1/1; base 1/1 | -| codex | kpersist-02 | 42,472 | 13,606 | +28,866 | +212.16% | skill 1/1; base 1/1 | -| codex | kpersist-03 | 70,384 | 86,760 | -16,376 | -18.88% | skill 1/1; base 1/1 | -| codex | kpersist-04 | 75,695 | 39,061 | +36,634 | +93.79% | skill 1/1; base 1/1 | -| codex | kpersist-05 | 41,998 | 73,582 | -31,584 | -42.92% | skill 1/1; base 1/1 | -| codex | kpersist-06 | 40,962 | 40,957 | +5 | +0.01% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,535,612 | 1,329,407 | +206,205 | +15.51% | skill 12/12; base 12/12 | +| claude-code | All cases | 1,141,507 | 648,162 | +493,345 | +76.11% | skill 6/6; base 6/6 | +| claude-code | kpersist-01 | 228,709 | 109,286 | +119,423 | +109.28% | skill 1/1; base 1/1 | +| claude-code | kpersist-02 | 228,638 | 107,440 | +121,198 | +112.81% | skill 1/1; base 1/1 | +| claude-code | kpersist-03 | 232,288 | 108,070 | +124,218 | +114.94% | skill 1/1; base 1/1 | +| claude-code | kpersist-04 | 236,289 | 108,684 | +127,605 | +117.41% | skill 1/1; base 1/1 | +| claude-code | kpersist-05 | 71,294 | 107,376 | -36,082 | -33.60% | skill 1/1; base 1/1 | +| claude-code | kpersist-06 | 144,289 | 107,306 | +36,983 | +34.46% | skill 1/1; base 1/1 | +| codex | All cases | 360,535 | 580,343 | -219,808 | -37.88% | skill 6/6; base 6/6 | +| codex | kpersist-01 | 81,456 | 124,764 | -43,308 | -34.71% | skill 1/1; base 1/1 | +| codex | kpersist-02 | 42,527 | 13,385 | +29,142 | +217.72% | skill 1/1; base 1/1 | +| codex | kpersist-03 | 80,182 | 306,156 | -225,974 | -73.81% | skill 1/1; base 1/1 | +| codex | kpersist-04 | 68,889 | 35,219 | +33,670 | +95.60% | skill 1/1; base 1/1 | +| codex | kpersist-05 | 42,468 | 59,767 | -17,299 | -28.94% | skill 1/1; base 1/1 | +| codex | kpersist-06 | 45,013 | 41,052 | +3,961 | +9.65% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,502,042 | 1,228,505 | +273,537 | +22.27% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-persistence/skill-card.md b/skills/kumo-model-persistence/skill-card.md index 40ba32c38..c9fe4a13c 100644 --- a/skills/kumo-model-persistence/skill-card.md +++ b/skills/kumo-model-persistence/skill-card.md @@ -9,14 +9,14 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill with coding agents to publish a Kumo model during its fit, reload an explicit registry version or retained bundle for scoring, and keep selected run files on a Snowflake stage after the notebook session ends.
+Developers and data scientists use this skill in Kumo Engine notebook workflows to publish trained models to a Snowflake model registry, reload an explicit model version or retained bundle for scoring without retraining, and retain or retrieve selected run diagnostics and output files through a Snowflake stage.
### Deployment Geography for Use:
Global
## Requirements / Dependencies:
-**Requires API Key or External Credential:** [Yes]
-**Credential Type(s):** [Other [Snowflake connection or Snowpark session of the user's notebook]]
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
@@ -25,18 +25,16 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/reuse.md](references/reuse.md)
-- [references/retention.md](references/retention.md)
-- [scripts/retain_artifacts.py](scripts/retain_artifacts.py)
+- [Publish and restore](references/reuse.md)
+- [Retain and retrieve artifacts](references/retention.md)
- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
-- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
## Skill Output:
-**Output Type(s):** [Code, Analysis]
-**Output Format:** [Markdown with inline Python code blocks; an archive with a manifest of byte lengths and SHA256 digests when artifacts are retained]
+**Output Type(s):** [Code, Analysis, Files]
+**Output Format:** [Markdown with inline Python code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Loading a model does not authorize training, replacing a registry version or overwriting a destination.]
+**Other Properties Related to Output:** [None]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -45,38 +43,44 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.10`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+6 evaluation tasks (5 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`); each task was also attempted without the skill as a baseline. Tier 1 static validation (11 validators), Tier 2 semantic deduplication (2 validators) and Tier 3 live agent evaluation (2 agents, 6 tasks) were run.
## Evaluation Metrics Used:
Reported benchmark dimensions:
- Security: Is it safe to use? Scored from `security` (100%).
- Correctness: Is the answer correct? Scored from `accuracy` (100%).
- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
-- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
-- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` (50%) and `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? `skill_efficiency` (50%) plus `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Accuracy: final-answer correctness against the reference answer.
-- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
-- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
-- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + ### Results at a Glance | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 89.4% — baseline ran, but no comparable score was available; uplift unavailable | 82.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 92.1% — baseline ran, but no comparable score was available; uplift unavailable | 80.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 91.7% → 100.0% (+8.3 points) | -| Correctness | 53.3% → 90.0% (+36.7 points) | 63.3% → 86.7% (+23.4 points) | -| Discoverability | 99.0% — baseline ran, but no comparable score was available; uplift unavailable | 91.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 28.1% → 72.2% (+44.1 points) | 28.9% → 50.6% (+21.7 points) | -| Efficiency | 85.9% — baseline ran, but no comparable score was available; uplift unavailable | 82.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 56.7% → 96.7% (+40.0 points) | 40.0% → 83.3% (+43.3 points) | +| Discoverability | 98.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 32.5% → 73.9% (+41.4 points) | 27.2% → 47.2% (+20.0 points) | +| Efficiency | 92.1% — baseline ran, but no comparable score was available; uplift unavailable | 82.5% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -91,28 +95,12 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.8% → 96.5% (-1.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.1% → 96.3% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 83.5% → 89.9% (+6.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | - -### Tier Status - -| Tier | Purpose | Status | Evidence | -|---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 22 finding(s) | -| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | - -### Overall Verdict - -✅ **Overall verdict: PASS — Recommended for publication** - -### Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +| codex | Efficiency | Token-only comparison | 84.7% → 89.7% (+5.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
-0.1.0 (source: pyproject.toml; git SHA 8aaa6cb3, committed 2026-11-18)
+0.1.0 (source: pyproject.toml)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-model-persistence/skill.oms.sig b/skills/kumo-model-persistence/skill.oms.sig index d48ab63ba..82fd4c666 100644 --- a/skills/kumo-model-persistence/skill.oms.sig +++ b/skills/kumo-model-persistence/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1wZXJzaXN0ZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJiMmRmMTZhMjIxMTAxZWQ4N2JkZDYwOTdmMTk1ZTUxYzJlMmVmZmEwN2NlNjRhMThhNjliM2VkMmYzMDNiZDJhIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2RkNjg1YzhlNmZjMDg1ODdhN2FhNWZkOWQzNjhhYTNhMDhjNTQ4M2FhZWQ0YTliNzBjZWE1MTE4Y2ExMzUzYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJiMWU4MWU2ODg2ZDNkYTczZmMwMzI2ZjhmMGQ1YTYzYTBjNzIzNjlkNmYyNWZkM2M0MjkyN2U5YWUyMDBlZTI4IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiM2JiZjZmNDg1OGNmNGEwMWY4NTAzMThjNTliNThmNDUzMTkwNzQ3NTI1OTc4ZGI2ZjIxYTA2MjRlOTBiYmRlZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3JldGVudGlvbi5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIyYjA1NGFmMzNmODE2MmM1MThkZWE5ZjdhZDg0NGMwZDQxNjA3YjAwM2Q1ODIxZDUwYTJiZjY4YmU2Y2FjZjMyIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcmV1c2UubWQiLAogICAgICAgICJkaWdlc3QiOiAiMmM1YTUwMzFjODUwMTlmMTIxNWU2Yzc0MGRjZWRlN2E2ZTkyMDg1MmIzZjU3MmYwNGY5NGRhMGVjY2M4YWU4MSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JldGFpbl9hcnRpZmFjdHMucHkiLAogICAgICAgICJkaWdlc3QiOiAiNGQzZDAzYjEwODlhMDI0MGQxZjkxM2MxZWQyZjhlNGFmOTg5ZWNiOGQzZTE5OWMzMmZkZGFlOTBmY2JhYWQzYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogImU0OThkNjY3MmIxNmM5NDA0NTdiZTVjYzU0NzljYzkxYzM4MzNhOTNjY2U5ZTJkOTE3ZDBkZjIzYzdlOTdmN2MiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDSbrAfQsoVPG4DRY9eYDMrvhlYQdA3cSL9C2wNpHjqjJOhwgYcJp+Qa3WoZW8hVSoCMQCQGVN4ym7lKLj+ruqEF266SAdrnF983/rGCZ7h8z+PaSl+VHHpNPk1Pelyu9TZNk4=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1wZXJzaXN0ZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJiN2UxY2FlOTk0N2RhNTI5ZmIwM2Q5M2I1NDQ4NzYwNjRkZjQ4NTExY2RiMzQ0ODVmY2FmNDQ4YjllZTFjZmQzIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiNzg4MWU2MjlmYzUzNjNlMTI2MTFmNDA1Yzg4YzU5ZjlkODk0YTExZWFlOTk5ZWNlZTY0MDAxNWQ5NWExMmY3MyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJiMWU4MWU2ODg2ZDNkYTczZmMwMzI2ZjhmMGQ1YTYzYTBjNzIzNjlkNmYyNWZkM2M0MjkyN2U5YWUyMDBlZTI4IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiM2JiZjZmNDg1OGNmNGEwMWY4NTAzMThjNTliNThmNDUzMTkwNzQ3NTI1OTc4ZGI2ZjIxYTA2MjRlOTBiYmRlZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3JldGVudGlvbi5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIyYjA1NGFmMzNmODE2MmM1MThkZWE5ZjdhZDg0NGMwZDQxNjA3YjAwM2Q1ODIxZDUwYTJiZjY4YmU2Y2FjZjMyIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcmV1c2UubWQiLAogICAgICAgICJkaWdlc3QiOiAiMmM1YTUwMzFjODUwMTlmMTIxNWU2Yzc0MGRjZWRlN2E2ZTkyMDg1MmIzZjU3MmYwNGY5NGRhMGVjY2M4YWU4MSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JldGFpbl9hcnRpZmFjdHMucHkiLAogICAgICAgICJkaWdlc3QiOiAiNGQzZDAzYjEwODlhMDI0MGQxZjkxM2MxZWQyZjhlNGFmOTg5ZWNiOGQzZTE5OWMzMmZkZGFlOTBmY2JhYWQzYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogImJjYzIxNjc2YjdiYmJkYTFlMjgzYmMwOTBmZTlhYzNmOTg5YzE3NDcyYjZkODM0OGRlMWRlM2I0YjQ5NzI0M2UiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMFeVa3Z/CPc/uXbxtNT/GKgFdtDCVUVpzN7l6x+BR21M5FMx4uIFY8WPzLTZvViK4gIwDG+C1VviG0kNZqvGtkS2E87P4RpF5lGhcCuS8Eu/VUU9Wg48YQErkqr3rkRYsE/w","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-tuning/BENCHMARK.md b/skills/kumo-model-tuning/BENCHMARK.md index c67ee01e1..a1172e504 100644 --- a/skills/kumo-model-tuning/BENCHMARK.md +++ b/skills/kumo-model-tuning/BENCHMARK.md @@ -10,7 +10,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-model-tuning` - Validation status: `passed` -- Evaluation date: 2026-11-18 +- Evaluation date: 2026-11-19 - Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 5 evaluation tasks (4 positive, 1 negative) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 99.7% — baseline ran, but no comparable score was available; uplift unavailable | 92.2% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 68.0% → 100.0% (+32.0 points) | 60.0% → 96.0% (+36.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 52.5% → 100.0% (+47.5 points) | 51.7% → 80.8% (+29.1 points) | -| Efficiency | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 91.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 99.8% — baseline ran, but no comparable score was available; uplift unavailable | 94.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | +| Correctness | 72.0% → 100.0% (+28.0 points) | 60.0% → 100.0% (+40.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 54.2% → 100.0% (+45.8 points) | 51.7% → 85.8% (+34.1 points) | +| Efficiency | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.2% → 97.3% (-0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.8% → 97.6% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.5% → 91.9% (+0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.9% → 92.3% (+1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 922,430 | 680,648 | +241,782 | +35.52% | skill 5/5; base 5/5 | -| claude-code | ktune-01 | 108,911 | 71,704 | +37,207 | +51.89% | skill 1/1; base 1/1 | -| claude-code | ktune-02 | 148,004 | 178,646 | -30,642 | -17.15% | skill 1/1; base 1/1 | -| claude-code | ktune-03 | 148,059 | 142,151 | +5,908 | +4.16% | skill 1/1; base 1/1 | -| claude-code | ktune-04 | 108,499 | 107,619 | +880 | +0.82% | skill 1/1; base 1/1 | -| claude-code | ktune-05 | 408,957 | 180,528 | +228,429 | +126.53% | skill 1/1; base 1/1 | -| codex | All cases | 265,904 | 280,821 | -14,917 | -5.31% | skill 5/5; base 5/5 | -| codex | ktune-01 | 41,800 | 63,362 | -21,562 | -34.03% | skill 1/1; base 1/1 | -| codex | ktune-02 | 56,900 | 54,890 | +2,010 | +3.66% | skill 1/1; base 1/1 | -| codex | ktune-03 | 56,919 | 54,763 | +2,156 | +3.94% | skill 1/1; base 1/1 | -| codex | ktune-04 | 46,208 | 62,868 | -16,660 | -26.50% | skill 1/1; base 1/1 | -| codex | ktune-05 | 64,077 | 44,938 | +19,139 | +42.59% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,188,334 | 961,469 | +226,865 | +23.60% | skill 10/10; base 10/10 | +| claude-code | All cases | 655,712 | 869,059 | -213,347 | -24.55% | skill 5/5; base 5/5 | +| claude-code | ktune-01 | 109,028 | 71,668 | +37,360 | +52.13% | skill 1/1; base 1/1 | +| claude-code | ktune-02 | 148,001 | 179,352 | -31,351 | -17.48% | skill 1/1; base 1/1 | +| claude-code | ktune-03 | 109,759 | 178,796 | -69,037 | -38.61% | skill 1/1; base 1/1 | +| claude-code | ktune-04 | 108,902 | 107,501 | +1,401 | +1.30% | skill 1/1; base 1/1 | +| claude-code | ktune-05 | 180,022 | 331,742 | -151,720 | -45.73% | skill 1/1; base 1/1 | +| codex | All cases | 262,464 | 260,559 | +1,905 | +0.73% | skill 5/5; base 5/5 | +| codex | ktune-01 | 41,815 | 57,196 | -15,381 | -26.89% | skill 1/1; base 1/1 | +| codex | ktune-02 | 72,462 | 55,253 | +17,209 | +31.15% | skill 1/1; base 1/1 | +| codex | ktune-03 | 57,242 | 54,477 | +2,765 | +5.08% | skill 1/1; base 1/1 | +| codex | ktune-04 | 45,814 | 48,744 | -2,930 | -6.01% | skill 1/1; base 1/1 | +| codex | ktune-05 | 45,131 | 44,889 | +242 | +0.54% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 918,176 | 1,129,618 | -211,442 | -18.72% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-tuning/skill-card.md b/skills/kumo-model-tuning/skill-card.md index 357412f57..4ee815a06 100644 --- a/skills/kumo-model-tuning/skill-card.md +++ b/skills/kumo-model-tuning/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill to configure and adjust Kumo Engine model plans (encoders, run mode, experiments, sampling, optimization, architecture, metrics and refits) for controlled experiments without changing task targets or evaluation protocol.
+Developers and data scientists use this skill to configure or adjust a Kumo model plan (encoders, run mode, experiment count, neighbor sampling, optimization, architecture, metrics and refits) for an existing predictive task while preserving its target and evaluation protocol.
### Deployment Geography for Use:
Global
@@ -31,7 +31,7 @@ Mitigation: Review and scan skill before deployment.
## Skill Output:
-**Output Type(s):** [Code, Configuration instructions, Analysis]
+**Output Type(s):** [Code, Configuration instructions]
**Output Format:** [Markdown with inline Python code blocks]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [None]
@@ -43,7 +43,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each attempt in its own isolated sandbox pod (`k8s-sandbox`); each task was also attempted without the skill as a baseline. Evaluator version `1.6.10`, evaluated 2026-11-18.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod; every task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -51,7 +51,7 @@ Reported benchmark dimensions:
- Correctness: Is the answer correct? Scored from `accuracy` (100%).
- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` (50%) and `behavior_check` (50%).
-- Efficiency: Did it avoid wasted tool calls and token usage? `skill_efficiency` (50%) + `token_efficiency` (50%); N/A sources are omitted, remaining weights renormalized, and the dimension marked partial.
+- Efficiency: Did it avoid wasted tool calls and token usage? `skill_efficiency` (50%) plus `token_efficiency` (50%).
Underlying evaluation signals used in this run:
- `security`: Unsafe operations, secret leakage, and unauthorized access.
@@ -65,7 +65,7 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
-> ✅ **Overall verdict: PASS — Recommended for publication** +**Overall verdict: PASS — Recommended for publication** **Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. @@ -73,29 +73,25 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 99.7% — baseline ran, but no comparable score was available; uplift unavailable | 92.2% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 68.0% → 100.0% (+32.0 points) | 60.0% → 96.0% (+36.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 52.5% → 100.0% (+47.5 points) | 51.7% → 80.8% (+29.1 points) | -| Efficiency | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 91.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 99.8% — baseline ran, but no comparable score was available; uplift unavailable | 94.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | +| Correctness | 72.0% → 100.0% (+28.0 points) | 60.0% → 100.0% (+40.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 54.2% → 100.0% (+45.8 points) | 51.7% → 85.8% (+34.1 points) | +| Efficiency | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | -**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. +Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. -Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. - -A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. - -#### Comparison details +### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.2% → 97.3% (-0.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.8% → 97.6% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 91.5% → 91.9% (+0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.9% → 92.3% (+1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
0.1.0 (source: pyproject.toml)
diff --git a/skills/kumo-model-tuning/skill.oms.sig b/skills/kumo-model-tuning/skill.oms.sig index 0cda032e5..75f93cb42 100644 --- a/skills/kumo-model-tuning/skill.oms.sig +++ b/skills/kumo-model-tuning/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC10dW5pbmciLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNjM1NzVkY2UzNTY2NzEzMGE1NTY1MjYyZDZlZmU1NDAyYWUxODFhNWUwOTVjM2NmMDM0MzE0YjQ5MjdhYTk0ZSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiM2I5MTRmN2ZiYTM5MDgyNDdiNDAwYjNjYzdkOWZmZjA1MGNkZTE5OTZiYmUyNWNkZmVkNTA3NjdhOTYwMjk2NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMTY0MDFkYmNmOTkzNGU2NTE5MjQ3YTQzYjdlZGE2MTc3OTY0YjEyZjcwYjY4NjE2NDFlNDQxNjBlNmQ4ZTdhYiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI1OGI2Y2YzMjI0MjUzZWVlNWY5YzFlMDNiMWY4MmUzYTk1ZWU4ZjFjNTI3YzI1Y2JiMzVjMTc1NDg5MGRkZmViIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiM2ViNDJiMGY4OTUzZDE3MDAyNTU4ZDA1YTg4Mjg1MTkzMWQ4Y2I1MTBjYThmN2U1YzYwZWI5Y2Y2NzkwMmM0ZCIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9wbGFucy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjE0MjhkZGQwYWIxZmE2MTBlOTFiZDFlMTg5YzE4Y2IzNmIyZDg3MjU2ZWIwYjg0YTU5YmFlZGViOTc5MjFhMjciLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCpPh9gERO0P1E4aX3WIkFPrXnsuPRDfZW6NGkZIo2nhyKiRT2l8aYjlTGJlDn0I/oCMQD61QYNU0U+6PcSubdDyV745hDnk19R+PmB1l2+B4R0d1ceSVaqIgZKJ/BBd6ytmRI=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC10dW5pbmciLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiM2RkY2Y3MzlmODE2YzdlNjRlZGUyZmViNmI5NDU2ODFmZDFlMmJiZGZiMWI1Njc3NDg2YWUyODQ1ZGJmNDFhYiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0sCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3NGE4ZTdjNjU2ZTEzMzlkYTQwMDJkMjc5ZmRhNzE5NjE3ZmJiNjVlMWEzOGY3NGViYmNiNGMyMjlhMmU4ZDM0IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxNjQwMWRiY2Y5OTM0ZTY1MTkyNDdhNDNiN2VkYTYxNzc5NjRiMTJmNzBiNjg2MTY0MWU0NDE2MGU2ZDhlN2FiIiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjU4YjZjZjMyMjQyNTNlZWU1ZjljMWUwM2IxZjgyZTNhOTVlZThmMWM1MjdjMjVjYmIzNWMxNzU0ODkwZGRmZWIiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzZWI0MmIwZjg5NTNkMTcwMDI1NThkMDVhODgyODUxOTMxZDhjYjUxMGNhOGY3ZTVjNjBlYjljZjY3OTAyYzRkIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BsYW5zLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMDJjODk0MDcyYTk3MGQ1YjEwNTI0Njg5ZjU2YzhjMmIzYWNhNmQ2NGFiZGMyNTAwODE2ZTMwNDhiZTRmZjlkZiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMC7ieYP+ZUWMcaEG7uCDuy5ynS4sgInN5TgIYLyNiPp3RJXg7xf9oaUJYYUaLPTodQIxAMnXqaj3BOFKMCfcYFdBXUDViZUTlWLh2wcxd7q6t3gGA4KUu8yaVriZsnuTgQsJiw==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-training-diagnostics/BENCHMARK.md b/skills/kumo-training-diagnostics/BENCHMARK.md index 377036542..ee12120e1 100644 --- a/skills/kumo-training-diagnostics/BENCHMARK.md +++ b/skills/kumo-training-diagnostics/BENCHMARK.md @@ -10,7 +10,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-training-diagnostics` - Validation status: `passed` -- Evaluation date: 2026-11-18 +- Evaluation date: 2026-11-19 - Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 5 evaluation tasks (4 positive, 1 negative) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 83.1% — baseline ran, but no comparable score was available; uplift unavailable | 88.2% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 48.0% → 80.0% (+32.0 points) | 32.0% → 84.0% (+52.0 points) | -| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 38.2% → 56.0% (+17.8 points) | 40.2% → 68.5% (+28.3 points) | -| Efficiency | 94.3% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 81.6% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | +| Correctness | 40.0% → 80.0% (+40.0 points) | 36.0% → 88.0% (+52.0 points) | +| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 40.7% → 48.5% (+7.8 points) | 50.2% → 63.2% (+13.0 points) | +| Efficiency | 94.4% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.5% → 97.0% (-1.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.4% → 97.2% (-1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 92.1% → 91.9% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.8% → 92.0% (+1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 758,267 | 536,081 | +222,186 | +41.45% | skill 5/5; base 5/5 | -| claude-code | kdiag-01 | 197,050 | 142,886 | +54,164 | +37.91% | skill 1/1; base 1/1 | -| claude-code | kdiag-02 | 197,230 | 107,354 | +89,876 | +83.72% | skill 1/1; base 1/1 | -| claude-code | kdiag-03 | 148,917 | 107,162 | +41,755 | +38.96% | skill 1/1; base 1/1 | -| claude-code | kdiag-04 | 107,579 | 107,407 | +172 | +0.16% | skill 1/1; base 1/1 | -| claude-code | kdiag-05 | 107,491 | 71,272 | +36,219 | +50.82% | skill 1/1; base 1/1 | -| codex | All cases | 274,654 | 179,075 | +95,579 | +53.37% | skill 5/5; base 5/5 | -| codex | kdiag-01 | 61,263 | 27,057 | +34,206 | +126.42% | skill 1/1; base 1/1 | -| codex | kdiag-02 | 62,133 | 40,982 | +21,151 | +51.61% | skill 1/1; base 1/1 | -| codex | kdiag-03 | 61,343 | 46,850 | +14,493 | +30.93% | skill 1/1; base 1/1 | -| codex | kdiag-04 | 44,384 | 46,477 | -2,093 | -4.50% | skill 1/1; base 1/1 | -| codex | kdiag-05 | 45,531 | 17,709 | +27,822 | +157.11% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,032,921 | 715,156 | +317,765 | +44.43% | skill 10/10; base 10/10 | +| claude-code | All cases | 640,415 | 537,025 | +103,390 | +19.25% | skill 5/5; base 5/5 | +| claude-code | kdiag-01 | 197,077 | 107,167 | +89,910 | +83.90% | skill 1/1; base 1/1 | +| claude-code | kdiag-02 | 155,096 | 107,338 | +47,758 | +44.49% | skill 1/1; base 1/1 | +| claude-code | kdiag-03 | 72,991 | 106,956 | -33,965 | -31.76% | skill 1/1; base 1/1 | +| claude-code | kdiag-04 | 107,809 | 144,238 | -36,429 | -25.26% | skill 1/1; base 1/1 | +| claude-code | kdiag-05 | 107,442 | 71,326 | +36,116 | +50.64% | skill 1/1; base 1/1 | +| codex | All cases | 273,653 | 222,237 | +51,416 | +23.14% | skill 5/5; base 5/5 | +| codex | kdiag-01 | 61,253 | 27,099 | +34,154 | +126.03% | skill 1/1; base 1/1 | +| codex | kdiag-02 | 79,572 | 48,830 | +30,742 | +62.96% | skill 1/1; base 1/1 | +| codex | kdiag-03 | 27,911 | 55,282 | -27,371 | -49.51% | skill 1/1; base 1/1 | +| codex | kdiag-04 | 59,760 | 45,959 | +13,801 | +30.03% | skill 1/1; base 1/1 | +| codex | kdiag-05 | 45,157 | 45,067 | +90 | +0.20% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 914,068 | 759,262 | +154,806 | +20.39% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-training-diagnostics/skill-card.md b/skills/kumo-training-diagnostics/skill-card.md index 530c205ea..de05202ed 100644 --- a/skills/kumo-training-diagnostics/skill-card.md +++ b/skills/kumo-training-diagnostics/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Evaluate an existing Kumo training run in a Python notebook using aggregate metrics, trial results, learning curves, refit evidence, and training progress; use for model-quality questions or training diagnostics, inspecting existing results without automatically tuning, rerunning, or deploying a model.
+Evaluate an existing Kumo training run in a Python notebook using aggregate metrics, trial results, learning curves, refit evidence, and training progress.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill to assess an existing Kumo training run in a notebook — reviewing available metrics, trial selection evidence, learning curves, refit results, and training progress — and to return an evidence-backed quality assessment without retraining or tuning.
+Developers and data scientists use this skill to answer model-quality and training-diagnostic questions about an existing Kumo training run in a Snowflake notebook, inspecting recorded metrics, trial selection, learning curves, and progress without tuning, rerunning, or deploying a model.
### Deployment Geography for Use:
Global
@@ -26,13 +26,12 @@ Mitigation: Review and scan skill before deployment.
## Reference(s):
- [Diagnostic workflow](references/diagnostics.md)
-- [Kumo model persistence skill](../kumo-model-persistence/SKILL.md)
- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
## Skill Output:
**Output Type(s):** [Analysis, Code]
-**Output Format:** [Markdown with inline Python code blocks]
+**Output Format:** [Markdown with inline Python notebook code blocks]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [None]
@@ -43,7 +42,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated `k8s-sandbox` pod; each task was also run without the skill as a baseline. Three-tier evaluation: Tier 1 static validation, Tier 2 semantic deduplication, Tier 3 live agent evaluation.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each run in its own isolated sandbox pod (`k8s-sandbox`); each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -65,22 +64,28 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
-**Overall verdict:** ✅ PASS — Recommended for publication +> ✅ **Overall verdict: PASS — Recommended for publication** -**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. ### Results at a Glance | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 83.1% — baseline ran, but no comparable score was available; uplift unavailable | 88.2% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 48.0% → 80.0% (+32.0 points) | 32.0% → 84.0% (+52.0 points) | -| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 38.2% → 56.0% (+17.8 points) | 40.2% → 68.5% (+28.3 points) | -| Efficiency | 94.3% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 81.6% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | +| Correctness | 40.0% → 80.0% (+40.0 points) | 36.0% → 88.0% (+52.0 points) | +| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 40.7% → 48.5% (+7.8 points) | 50.2% → 63.2% (+13.0 points) | +| Efficiency | 94.4% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. -Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. ### Comparison details @@ -89,9 +94,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.5% → 97.0% (-1.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.4% → 97.2% (-1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 92.1% → 91.9% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.8% → 92.0% (+1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
0.1.0 (source: pyproject.toml)
diff --git a/skills/kumo-training-diagnostics/skill.oms.sig b/skills/kumo-training-diagnostics/skill.oms.sig index 89ca57bf4..bb7cdb6ee 100644 --- a/skills/kumo-training-diagnostics/skill.oms.sig +++ b/skills/kumo-training-diagnostics/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby10cmFpbmluZy1kaWFnbm9zdGljcyIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIwMjJkMjkxYzMwYTkzMGE1YTkyMjhhOWY0Yzk2MTZjZTlhMjIxY2Y2M2NkYzY3NjgyNzAwNWM1MjNhYjc0NzU0IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJkMDljYjdiNGY4MGRjNTdiZjEwOTY3MDYxNTJkNTQ4N2YyZGExYWRhNDdiYjY3OGQxYjg2MzI1NTlkN2JmYzA3IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI2NGRkODhhZTY2MDJmYjFmMjdhNGY0M2E0ZTQyZDYwN2Y3M2ZkOWYxN2FhNDg4MTVhMzhjZTlmNDM0MGZlZjc2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjhjNGM3YjNhNjlkZDE0NzFhMDNiMjU3M2I3YmVlMTk0MTEyOWI3YTUyYzg5OThlYjg5NzcyMmRmMjE1NTI5NTQiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlNzVkMzBlNTQ1NTMyMmY4NWM1NDBkOGI0MzdkYjBjMDA3NjFkNTI3NThhNTJmM2I5YThhNzY4MDM2MWZjZTMwIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2RpYWdub3N0aWNzLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMWIwNTQ3MzBjNjkyNTJjYzU5ZjFkM2M4ZDhmNDA3MDYwYmNmYTI1NDk3N2QyZmViNTA3MWNmY2E5ZDVjNzc1YiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMFhuql36zE6dgiSvxKKu6C8432nO2F4DH1bIAHf5V1Xz0Ad/1AzTyPx/ECsF+X7mLAIxAL7fIQb6woGh39UigiSihRZ/HP644Q7519C7gqqiJEasBZnWIcQsY6bWNvDyxBkVGA==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby10cmFpbmluZy1kaWFnbm9zdGljcyIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIyODUzMTE0ODk5MmU1MTYyY2I5YmJhMjRkMmNhN2M1ZjVlNWY4MDNiZGQwYzdlNDFkNTg1MjcwZjkxN2UxNDVmIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI5Mjg0ZDI0N2UwMmQxYTljMGE4MzBkNjk3MmVmMjdhODM0NDliZjM5MjkzMjkxM2U4ODIzNzViZTI1OTIxY2JhIiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI2NGRkODhhZTY2MDJmYjFmMjdhNGY0M2E0ZTQyZDYwN2Y3M2ZkOWYxN2FhNDg4MTVhMzhjZTlmNDM0MGZlZjc2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjhjNGM3YjNhNjlkZDE0NzFhMDNiMjU3M2I3YmVlMTk0MTEyOWI3YTUyYzg5OThlYjg5NzcyMmRmMjE1NTI5NTQiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlNzVkMzBlNTQ1NTMyMmY4NWM1NDBkOGI0MzdkYjBjMDA3NjFkNTI3NThhNTJmM2I5YThhNzY4MDM2MWZjZTMwIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2RpYWdub3N0aWNzLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMWNlMGIxOGUyY2U2NjdiNTNkNDhiZTFiOWY5YmQwOWJmNDY3YWNiNTJjM2UxMjRiZWI5NjEzMDQ5MmU3OGQ2YiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMESrJBwoNsJnGplr9bI7NBkUoRZtjpYF1mVuXDhoNyYSHUg1PR7gLHTWRwTswxAwDwIwbKG4AoDyJRMUvN2rC9LAcFjiOCuodOpuOBaY0n+JB1iUGPfnxj9QnvSNFsalDkjs","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumoai-reference/BENCHMARK.md b/skills/kumoai-reference/BENCHMARK.md index 800ab41ba..ece5aee44 100644 --- a/skills/kumoai-reference/BENCHMARK.md +++ b/skills/kumoai-reference/BENCHMARK.md @@ -10,7 +10,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumoai-reference` - Validation status: `passed` -- Evaluation date: 2026-11-18 +- Evaluation date: 2026-11-19 - Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 5 evaluation tasks (4 positive, 1 negative) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.2% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 90.0% → 80.0% (-10.0 points) | -| Correctness | 68.0% → 100.0% (+32.0 points) | 84.0% → 96.0% (+12.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 88.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 50.5% → 78.0% (+27.5 points) | 58.0% → 84.7% (+26.7 points) | -| Efficiency | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | 85.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | 93.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 90.0% → 100.0% (+10.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 84.0% → 100.0% (+16.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 50.2% → 82.9% (+32.7 points) | 60.0% → 89.3% (+29.3 points) | +| Efficiency | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | 84.3% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 94.3% → 87.8% (-6.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 95.4% → 88.4% (-7.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 77.4% → 87.0% (+9.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 78.9% → 86.7% (+7.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,351,652 | 1,272,515 | +79,137 | +6.22% | skill 5/5; base 5/5 | -| claude-code | kref-01 | 121,660 | 35,613 | +86,047 | +241.62% | skill 1/1; base 1/1 | -| claude-code | kref-02 | 523,009 | 148,685 | +374,324 | +251.76% | skill 1/1; base 1/1 | -| claude-code | kref-03 | 233,277 | 306,293 | -73,016 | -23.84% | skill 1/1; base 1/1 | -| claude-code | kref-04 | 325,840 | 109,513 | +216,327 | +197.54% | skill 1/1; base 1/1 | -| claude-code | kref-05 | 147,866 | 672,411 | -524,545 | -78.01% | skill 1/1; base 1/1 | -| codex | All cases | 825,268 | 673,781 | +151,487 | +22.48% | skill 5/5; base 5/5 | -| codex | kref-01 | 69,115 | 28,809 | +40,306 | +139.91% | skill 1/1; base 1/1 | -| codex | kref-02 | 280,824 | 252,624 | +28,200 | +11.16% | skill 1/1; base 1/1 | -| codex | kref-03 | 120,693 | 91,963 | +28,730 | +31.24% | skill 1/1; base 1/1 | -| codex | kref-04 | 137,479 | 47,353 | +90,126 | +190.33% | skill 1/1; base 1/1 | -| codex | kref-05 | 217,157 | 253,032 | -35,875 | -14.18% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 2,176,920 | 1,946,296 | +230,624 | +11.85% | skill 10/10; base 10/10 | +| claude-code | All cases | 1,436,485 | 1,128,611 | +307,874 | +27.28% | skill 5/5; base 5/5 | +| claude-code | kref-01 | 121,632 | 35,554 | +86,078 | +242.10% | skill 1/1; base 1/1 | +| claude-code | kref-02 | 534,256 | 188,249 | +346,007 | +183.80% | skill 1/1; base 1/1 | +| claude-code | kref-03 | 265,314 | 314,517 | -49,203 | -15.64% | skill 1/1; base 1/1 | +| claude-code | kref-04 | 365,817 | 110,088 | +255,729 | +232.30% | skill 1/1; base 1/1 | +| claude-code | kref-05 | 149,466 | 480,203 | -330,737 | -68.87% | skill 1/1; base 1/1 | +| codex | All cases | 756,110 | 575,543 | +180,567 | +31.37% | skill 5/5; base 5/5 | +| codex | kref-01 | 69,061 | 38,017 | +31,044 | +81.66% | skill 1/1; base 1/1 | +| codex | kref-02 | 219,983 | 134,324 | +85,659 | +63.77% | skill 1/1; base 1/1 | +| codex | kref-03 | 97,023 | 127,096 | -30,073 | -23.66% | skill 1/1; base 1/1 | +| codex | kref-04 | 119,192 | 48,860 | +70,332 | +143.95% | skill 1/1; base 1/1 | +| codex | kref-05 | 250,851 | 227,246 | +23,605 | +10.39% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,192,595 | 1,704,154 | +488,441 | +28.66% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -109,15 +109,15 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c
Show detailed findings and successful checks +- **HIGH** DUPLICATE/duplicate: Duplicate content found across references/graph_construction.md and references/kumo_finetuning_sdk.md: + "### Defining tables" in references/graph_construction.md (lines 106-143) + vs "## 4. Tables" in references/kumo_finetuning_sdk.md (lines 308-354) (`references/graph_construction.md:106`) - **HIGH** DUPLICATE/duplicate: Duplicate content found across references/kumo_finetuning_sdk.md and references/pql_writing.md and references/task_journeys.md: "## 6. Predictive query" in references/kumo_finetuning_sdk.md (lines 393-414) vs "## Task type is inferred from query shape" in references/pql_writing.md (lines 202-220) vs "## SDK usage" in references/pql_writing.md (lines 438-471) vs "### Required pre-flight, before any training step" in references/pql_writing.md (lines 472-490) vs "## Classification and regression" in references/task_journeys.md (lines 9-26) (`references/kumo_finetuning_sdk.md:393`) -- **HIGH** DUPLICATE/duplicate: Duplicate content found across references/graph_construction.md and references/kumo_finetuning_sdk.md: - "### Defining tables" in references/graph_construction.md (lines 106-143) - vs "## 4. Tables" in references/kumo_finetuning_sdk.md (lines 308-354) (`references/graph_construction.md:106`) - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumoai-reference/SKILL.md`) - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/kumoai-reference/SKILL.md`) - **MEDIUM** QUALITY/quality_efficiency: Deeply nested references in outputs.md (`skills/kumoai-reference/SKILL.md`) diff --git a/skills/kumoai-reference/skill-card.md b/skills/kumoai-reference/skill-card.md index 4645fceff..efc46b7ac 100644 --- a/skills/kumoai-reference/skill-card.md +++ b/skills/kumoai-reference/skill-card.md @@ -9,14 +9,14 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill, normally through the kumo-data-science skill, so a coding agent writes Kumo graph definitions, predictive queries, and training and prediction code that match the shipped Kumo SDK.
+Developers and data scientists use this skill, after a request passes kumo-data-science intake checks, to generate Kumo table/graph definitions, PQL predictive queries, and version-matched Kumo SDK training and prediction code for the local in-process Kumo runtime.
### Deployment Geography for Use:
Global
## Requirements / Dependencies:
-**Requires API Key or External Credential:** [Optional]
-**Credential Type(s):** [Cloud Credentials, OAuth Token, Other [Snowflake connections.toml entry or key pair]]
+**Requires API Key or External Credential:** [Yes]
+**Credential Type(s):** [Cloud Credentials, Other [Snowflake runtime token, connections.toml entry, or key pair from a secret store]]
Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
@@ -25,21 +25,20 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/graph_construction.md](references/graph_construction.md)
-- [references/pql_writing.md](references/pql_writing.md)
-- [references/kumo_finetuning_sdk.md](references/kumo_finetuning_sdk.md)
-- [references/task_journeys.md](references/task_journeys.md)
-- [references/outputs.md](references/outputs.md)
-- [references/troubleshooting_signals.md](references/troubleshooting_signals.md)
+- [Graph construction](references/graph_construction.md)
+- [PQL writing](references/pql_writing.md)
+- [Kumo SDK code generation reference](references/kumo_finetuning_sdk.md)
+- [Task journeys](references/task_journeys.md)
+- [Predictions, embeddings and destinations](references/outputs.md)
+- [Troubleshooting signals](references/troubleshooting_signals.md)
- [NVIDIA/kumo-engine#134 (graph visualization)](https://github.com/NVIDIA/kumo-engine/pull/134)
-- [NVIDIA/kumo-engine](https://github.com/NVIDIA/kumo-engine)
## Skill Output:
-**Output Type(s):** [Code, Analysis]
-**Output Format:** [Markdown with inline Python and PQL code blocks]
+**Output Type(s):** [Code, Configuration instructions]
+**Output Format:** [Markdown with inline Python code blocks and PQL statements]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Generates code only; does not run jobs, interpret metrics, or repair runtime errors.]
+**Other Properties Related to Output:** [Generated code uses only SDK signatures documented in the skill references; it does not run jobs or interpret results.]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -48,24 +47,24 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment; evaluator version `1.6.10`; evaluated 2026-11-18). Each task was also attempted without the skill as a baseline.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated `k8s-sandbox` pod; each task also run without the skill as a baseline. Evaluated 2026-11-19 with evaluator version `1.6.10`.
## Evaluation Metrics Used:
Reported benchmark dimensions:
- Security: Is it safe to use? Scored from `security` (100%).
- Correctness: Is the answer correct? Scored from `accuracy` (100%).
- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
-- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
-- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+- Effectiveness: Did the skill help complete the task? `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Accuracy: final-answer correctness against the reference answer.
-- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
-- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
-- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
@@ -80,12 +79,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.2% — baseline ran, but no comparable score was available; uplift unavailable | 87.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 90.0% → 80.0% (-10.0 points) | -| Correctness | 68.0% → 100.0% (+32.0 points) | 84.0% → 96.0% (+12.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 88.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 50.5% → 78.0% (+27.5 points) | 58.0% → 84.7% (+26.7 points) | -| Efficiency | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | 85.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | 93.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 90.0% → 100.0% (+10.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 84.0% → 100.0% (+16.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 50.2% → 82.9% (+32.7 points) | 60.0% → 89.3% (+29.3 points) | +| Efficiency | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | 84.3% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -93,16 +92,16 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. -#### Comparison details +### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 94.3% → 87.8% (-6.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 95.4% → 88.4% (-7.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 77.4% → 87.0% (+9.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 78.9% → 86.7% (+7.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -113,7 +112,7 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | ## Skill Version(s):
-0.1.0 (source: pyproject.toml; git SHA 8aaa6cb3, committed 2026-11-18)
+0.1.0 (source: pyproject.toml); git SHA bfbf737f, committed 2026-11-18
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumoai-reference/skill.oms.sig b/skills/kumoai-reference/skill.oms.sig index 55a39a66e..73ce3b80c 100644 --- a/skills/kumoai-reference/skill.oms.sig +++ b/skills/kumoai-reference/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtb2FpLXJlZmVyZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI3YmMzMjU5ODMxYjdhODViOGI2YzhmYWQ0YjU4OWNjNDkxYzczODUwYzdiZmFjMGQ3ZTZkZmFmZmJmY2FiNzY2IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzEwMjczMjkxNWMwMGQ0MzhjNTFhMDUxZGRiMWEzY2U2ZTZhM2RmNjA3ZGFmZDI1MDk4ZWIwYTAwMjdiNzAzOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMDQzMTYyYmFiYzc1NWFiYTEzNTgyNWFmM2NhMzc1ZDhmMGQ1MWQ5MDRjZGNhYmU2M2Y3MzlhNjNmODRjZDRhZiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlODBjYWNhY2NjNjA3YjIwMDFmYzE5MDQ0ZGNkNzEwOTJiNWEwNTQzYmY1NzYzMDJkZmRmMDgxMmY0NzhkZTZjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOGU4OGUyM2IxN2EyMGUxN2E5Mjc4Njg3NDJmYzQwNTA2NGE4OGQxNDVmMmNiODgzNmE4NTJlMzQwMDVlNGJiYyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvZ3JhcGhfY29uc3RydWN0aW9uLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlMDQxYWY1ZTkyZmI2NmYyMjE1NWU0NTk0Mjk5NmFlZjM4ZWFjNGE0Y2E3ZDE0YjlkOWNhZWU4ZWFmNmI0OWI4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9rdW1vX2ZpbmV0dW5pbmdfc2RrLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJmMjlmM2QyZDJiY2E5NTUwZGU1MjViYjViYzYyZTU1ODc4ZTBmNTRiYTdmNWYxMDliMDJmYzk3OGJiYjg1ZWJhIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9vdXRwdXRzLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI0NTc4N2RhY2I4ZTk3NzJhZTk1YjA1NzRmMjk3ODM0ODdlNGNjYzliNzBhZWUxYzkwNGQ4MGVkMmU4NWMxM2FmIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9wcWxfd3JpdGluZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzA5YzIzYmExMGM3YWEwMDg5OTY0NjAwMGExNmVkYjMyYzg5ZDI5YzYzNTllNjE0ZGM5YWM2YTJlYTkyMGYwOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdGFza19qb3VybmV5cy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNzliNTU1ZGJmMmQ0N2MyOThlY2U2NGY0ZjVkODg0ZGU3N2Q0Zjc2YTY1M2JhOWUyMTgzMzBjM2YxYzE0N2ZiNSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdHJvdWJsZXNob290aW5nX3NpZ25hbHMubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjA3OTAyMjU1ZDhlNTIwYzc0ZmMxOThmZTBhZWQzMzEyNGJmNDhhYWI5MTlkOWEzZTZkZmI5OTY3YjdkMjU0YzMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMGj8hOUtRh5Vg8qQTMqDM6OX/FAA0TaoIvkVdpKr88BMB02os0wVr9SXrGR0QfXfXAIwC/VDydWUPVbxU3fz4sfOkcw0G5h7SGynZ7KOznRD5fn12P75DJlrrekf+LCh3g1y","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtb2FpLXJlZmVyZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI2Nzc5N2VlY2JmMThhN2NlOGViYmRhYTI0OGMyNDNjM2MyYTQ0ZmM5MGY2ZGFmODE1NDg1OGY3YjY2Y2RkMjUzIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIzNTg0ZjBiNjYyNzE1MDhkYjMwNDYwOGU0MzkxZjkxZjU3ZGE1MDBiYzcwYWM3MzVmZjM2Y2JhZTczODc1YWU3IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIwNDMxNjJiYWJjNzU1YWJhMTM1ODI1YWYzY2EzNzVkOGYwZDUxZDkwNGNkY2FiZTYzZjczOWE2M2Y4NGNkNGFmIiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImU4MGNhY2FjY2M2MDdiMjAwMWZjMTkwNDRkY2Q3MTA5MmI1YTA1NDNiZjU3NjMwMmRmZGYwODEyZjQ3OGRlNmMiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI4ZTg4ZTIzYjE3YTIwZTE3YTkyNzg2ODc0MmZjNDA1MDY0YTg4ZDE0NWYyY2I4ODM2YTg1MmUzNDAwNWU0YmJjIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2dyYXBoX2NvbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImUwNDFhZjVlOTJmYjY2ZjIyMTU1ZTQ1OTQyOTk2YWVmMzhlYWM0YTRjYTdkMTRiOWQ5Y2FlZThlYWY2YjQ5YjgiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMva3Vtb19maW5ldHVuaW5nX3Nkay5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImYyOWYzZDJkMmJjYTk1NTBkZTUyNWJiNWJjNjJlNTU4NzhlMGY1NGJhN2Y1ZjEwOWIwMmZjOTc4YmJiODVlYmEiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvb3V0cHV0cy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjQ1Nzg3ZGFjYjhlOTc3MmFlOTViMDU3NGYyOTc4MzQ4N2U0Y2NjOWI3MGFlZTFjOTA0ZDgwZWQyZTg1YzEzYWYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcHFsX3dyaXRpbmcubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJjMDljMjNiYTEwYzdhYTAwODk5NjQ2MDAwYTE2ZWRiMzJjODlkMjljNjM1OWU2MTRkYzlhYzZhMmVhOTIwZjA4IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3Rhc2tfam91cm5leXMubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI3OWI1NTVkYmYyZDQ3YzI5OGVjZTY0ZjRmNWQ4ODRkZTc3ZDRmNzZhNjUzYmE5ZTIxODMzMGMzZjFjMTQ3ZmI1IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3Ryb3VibGVzaG9vdGluZ19zaWduYWxzLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNmNhZmZlOTZmYWVjZmYwMTZjMWJiOTI0YjE2MTUyMzUzNGJkYWJiZDI1OGMwODNmYTVhMjRmYjc1M2QwYWJmYiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMGJA7BJyARc0pRTO14QTIRps54brXXTDCWPpVBvQqg//jqQ4nG896eCliSFndyO2ewIxANDI/oL2o387XCIGYC6eEZkQ4BlMFg95N6Eh/aouL2h5nGrymgvH35O2WdGc6878wA==","keyid":""}]}} \ No newline at end of file From 2bbedbe65a20f3af75d0b430c096c9f8c577a305 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Wed, 18 Nov 2026 18:45:14 -0600 Subject: [PATCH 34/35] agent-skills: null CSV tokens in the time column only; retry %z without it strings_can_be_null=True nulled Arrow's null tokens (NA, null, N/A) in every string column, so an entity key such as `NA` staged as null and no longer matched the graph's key. The option is gone; the tokens are nulled in the time column only, before it is parsed. Under a %z format, a value without an offset raised, while the graph retries it without the directive. Staging now does the same, value by value, and still raises on a value that fits neither. Co-Authored-By: Claude Opus 5.5 --- agent-skills/skills/hosts/generic/runtime.md | 32 +++++++++++++------ skills/kumo-data-science/SKILL.md | 2 +- .../hosts/generic/runtime.md | 32 +++++++++++++------ 3 files changed, 47 insertions(+), 19 deletions(-) diff --git a/agent-skills/skills/hosts/generic/runtime.md b/agent-skills/skills/hosts/generic/runtime.md index 558400fc9..398fee17c 100644 --- a/agent-skills/skills/hosts/generic/runtime.md +++ b/agent-skills/skills/hosts/generic/runtime.md @@ -340,9 +340,10 @@ system; the destination may be in another system (`adapter.md` §4) and plays no if ENTITY_TIME_FORMAT and not re.search(r'%[-_0^#.:+\d]|%f', ENTITY_TIME_FORMAT) else None) source_format = ENTITY_FORMAT - if ENTITY_FORMAT == 'csv' and ENTITY_TIME and ARROW_FORMAT: # text for the format; '' is null + if ENTITY_FORMAT == 'csv' and ENTITY_TIME and ARROW_FORMAT: # text, parsed with the format below source_format = ds.CsvFileFormat(convert_options=pyarrow.csv.ConvertOptions( - column_types={ENTITY_TIME: pa.string()}, strings_can_be_null=True)) + column_types={ENTITY_TIME: pa.string()})) + NULL_TOKENS = pa.array(pyarrow.csv.ConvertOptions().null_values) # '', 'NA', 'null', ... scanner = ds.dataset(source_path, format=source_format, filesystem=source_fs).scanner(columns=list(keep)) @@ -353,10 +354,21 @@ system; the destination may be in another system (`adapter.md` §4) and plays no for f in scanner.projected_schema]) def to_column(col, field): # text in the entity's format reads as the graph reads it - if (field.name == 'TIMESTAMP' and ARROW_FORMAT + if not (field.name == 'TIMESTAMP' and ARROW_FORMAT and (pa.types.is_string(col.type) or pa.types.is_large_string(col.type))): - return pc.strptime(col, format=ARROW_FORMAT, unit=field.type.unit) # raises on a misfit - return pc.cast(col, field.type) + return pc.cast(col, field.type) + unit = field.type.unit + col = pc.if_else(pc.is_in(col, value_set=NULL_TOKENS), pa.scalar(None, col.type), col) + if '%z' not in ARROW_FORMAT: + return pc.strptime(col, format=ARROW_FORMAT, unit=unit) # raises on a misfit + # As the graph does, a value without an offset is read by the format without %z. + out = pc.coalesce( + pc.strptime(col, format=ARROW_FORMAT, unit=unit, error_is_null=True), + pc.strptime(col, format=ARROW_FORMAT.replace('%z', ''), unit=unit, error_is_null=True)) + misfit = pc.and_(pc.is_valid(col), pc.is_null(out)) + if pc.any(misfit).as_py(): + raise pa.ArrowInvalid(f'{pc.filter(col, misfit)[0]} does not fit {ARROW_FORMAT}') + return out pop_fs, pop_dir = arrow_fs(POPULATION_PATH) if isinstance(pop_fs, pyarrow.fs.LocalFileSystem): @@ -384,12 +396,14 @@ system; the destination may be in another system (`adapter.md` §4) and plays no is the `timestamp_format` the entity `Table` declares on its time column (`kumoai-reference/references/graph_construction.md`, *A time column must be a real date*), else `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The format is - Kumo's (chrono) syntax. When Arrow can parse it too, a CSV time column is read as text, with an - empty cell as null, and parsed with the format, because ISO-looking text such as `2024-01-02` - under `%Y-%d-%m` means what the format says, as it does in the graph. Arrow cannot parse + Kumo's (chrono) syntax. When Arrow can parse it too, a CSV time column is read as text and + parsed with the format, because ISO-looking text such as `2024-01-02` under `%Y-%d-%m` means + what the format says, as it does in the graph; a cell the CSV reader takes as null (empty, + `NA`, `null`) stays null, and no other column changes. Arrow cannot parse chrono's own directives (`%.f`, `%-m`, `%:z`); with one of those the column is read and cast as without a format, which keeps the type the reader inferred: nanoseconds, or a UTC offset as a - time zone. If the parse or the cast fails, report the format and the failing value rather than + time zone. Under `%z`, a value without an offset is read without it, as the graph reads it. + If the parse or the cast fails, report the format and the failing value rather than guessing another format. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed diff --git a/skills/kumo-data-science/SKILL.md b/skills/kumo-data-science/SKILL.md index c4bef817d..6aae6b86b 100644 --- a/skills/kumo-data-science/SKILL.md +++ b/skills/kumo-data-science/SKILL.md @@ -7,7 +7,7 @@ description: "Use for KumoAI graphs and predictive models on relational data in ## Build marker -**`SKILL-BUILD: 2026-11-18-7218a192-dirty`** +**`SKILL-BUILD: 2026-11-19-bef5ab4e-dirty`** Quote this string verbatim whenever asked which build of the skill is loaded. If you cannot read it from this file, you have not loaded the skill — say so rather than guessing. diff --git a/skills/kumo-data-science/hosts/generic/runtime.md b/skills/kumo-data-science/hosts/generic/runtime.md index 558400fc9..398fee17c 100644 --- a/skills/kumo-data-science/hosts/generic/runtime.md +++ b/skills/kumo-data-science/hosts/generic/runtime.md @@ -340,9 +340,10 @@ system; the destination may be in another system (`adapter.md` §4) and plays no if ENTITY_TIME_FORMAT and not re.search(r'%[-_0^#.:+\d]|%f', ENTITY_TIME_FORMAT) else None) source_format = ENTITY_FORMAT - if ENTITY_FORMAT == 'csv' and ENTITY_TIME and ARROW_FORMAT: # text for the format; '' is null + if ENTITY_FORMAT == 'csv' and ENTITY_TIME and ARROW_FORMAT: # text, parsed with the format below source_format = ds.CsvFileFormat(convert_options=pyarrow.csv.ConvertOptions( - column_types={ENTITY_TIME: pa.string()}, strings_can_be_null=True)) + column_types={ENTITY_TIME: pa.string()})) + NULL_TOKENS = pa.array(pyarrow.csv.ConvertOptions().null_values) # '', 'NA', 'null', ... scanner = ds.dataset(source_path, format=source_format, filesystem=source_fs).scanner(columns=list(keep)) @@ -353,10 +354,21 @@ system; the destination may be in another system (`adapter.md` §4) and plays no for f in scanner.projected_schema]) def to_column(col, field): # text in the entity's format reads as the graph reads it - if (field.name == 'TIMESTAMP' and ARROW_FORMAT + if not (field.name == 'TIMESTAMP' and ARROW_FORMAT and (pa.types.is_string(col.type) or pa.types.is_large_string(col.type))): - return pc.strptime(col, format=ARROW_FORMAT, unit=field.type.unit) # raises on a misfit - return pc.cast(col, field.type) + return pc.cast(col, field.type) + unit = field.type.unit + col = pc.if_else(pc.is_in(col, value_set=NULL_TOKENS), pa.scalar(None, col.type), col) + if '%z' not in ARROW_FORMAT: + return pc.strptime(col, format=ARROW_FORMAT, unit=unit) # raises on a misfit + # As the graph does, a value without an offset is read by the format without %z. + out = pc.coalesce( + pc.strptime(col, format=ARROW_FORMAT, unit=unit, error_is_null=True), + pc.strptime(col, format=ARROW_FORMAT.replace('%z', ''), unit=unit, error_is_null=True)) + misfit = pc.and_(pc.is_valid(col), pc.is_null(out)) + if pc.any(misfit).as_py(): + raise pa.ArrowInvalid(f'{pc.filter(col, misfit)[0]} does not fit {ARROW_FORMAT}') + return out pop_fs, pop_dir = arrow_fs(POPULATION_PATH) if isinstance(pop_fs, pyarrow.fs.LocalFileSystem): @@ -384,12 +396,14 @@ system; the destination may be in another system (`adapter.md` §4) and plays no is the `timestamp_format` the entity `Table` declares on its time column (`kumoai-reference/references/graph_construction.md`, *A time column must be a real date*), else `None`: a CSV reads `03/15/2024` as text, and a plain cast parses only ISO-8601. The format is - Kumo's (chrono) syntax. When Arrow can parse it too, a CSV time column is read as text, with an - empty cell as null, and parsed with the format, because ISO-looking text such as `2024-01-02` - under `%Y-%d-%m` means what the format says, as it does in the graph. Arrow cannot parse + Kumo's (chrono) syntax. When Arrow can parse it too, a CSV time column is read as text and + parsed with the format, because ISO-looking text such as `2024-01-02` under `%Y-%d-%m` means + what the format says, as it does in the graph; a cell the CSV reader takes as null (empty, + `NA`, `null`) stays null, and no other column changes. Arrow cannot parse chrono's own directives (`%.f`, `%-m`, `%:z`); with one of those the column is read and cast as without a format, which keeps the type the reader inferred: nanoseconds, or a UTC offset as a - time zone. If the parse or the cast fails, report the format and the failing value rather than + time zone. Under `%z`, a value without an offset is read without it, as the graph reads it. + If the parse or the cast fails, report the format and the failing value rather than guessing another format. The population is written as Parquet either way. `POPULATION_PATH` is the confirmed path, ending in `/`, in the source's file system or bucket, and `POPULATION_CONFIRMED` the same path as Node 1d confirmed From 7c560174644947f7086172e83b838f92e62e0274 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Thu, 19 Nov 2026 01:36:56 +0000 Subject: [PATCH 35/35] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/kumo-custom-populations/BENCHMARK.md | 42 ++++++------ skills/kumo-custom-populations/skill-card.md | 32 +++++---- skills/kumo-custom-populations/skill.oms.sig | 2 +- skills/kumo-data-science/BENCHMARK.md | 60 ++++++++-------- skills/kumo-data-science/skill-card.md | 68 +++++++++---------- skills/kumo-data-science/skill.oms.sig | 2 +- skills/kumo-graph-visualization/BENCHMARK.md | 40 +++++------ skills/kumo-graph-visualization/skill-card.md | 34 ++++------ skills/kumo-graph-visualization/skill.oms.sig | 2 +- skills/kumo-model-explanations/BENCHMARK.md | 42 ++++++------ skills/kumo-model-explanations/skill-card.md | 12 ++-- skills/kumo-model-explanations/skill.oms.sig | 2 +- skills/kumo-model-persistence/BENCHMARK.md | 46 ++++++------- skills/kumo-model-persistence/skill-card.md | 16 ++--- skills/kumo-model-persistence/skill.oms.sig | 2 +- skills/kumo-model-tuning/BENCHMARK.md | 40 +++++------ skills/kumo-model-tuning/skill-card.md | 36 +++++++--- skills/kumo-model-tuning/skill.oms.sig | 2 +- skills/kumo-training-diagnostics/BENCHMARK.md | 40 +++++------ .../kumo-training-diagnostics/skill-card.md | 39 +++++++---- .../kumo-training-diagnostics/skill.oms.sig | 2 +- skills/kumoai-reference/BENCHMARK.md | 42 ++++++------ skills/kumoai-reference/skill-card.md | 18 ++--- skills/kumoai-reference/skill.oms.sig | 2 +- 24 files changed, 324 insertions(+), 299 deletions(-) diff --git a/skills/kumo-custom-populations/BENCHMARK.md b/skills/kumo-custom-populations/BENCHMARK.md index edfa72e57..b86790de1 100644 --- a/skills/kumo-custom-populations/BENCHMARK.md +++ b/skills/kumo-custom-populations/BENCHMARK.md @@ -10,7 +10,7 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `kumo-custom-populations` - Validation status: `passed` -- Evaluation date: 2026-11-18 +- Evaluation date: 2026-11-19 - Evaluator version: `1.6.10` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 5 evaluation tasks (4 positive, 1 negative) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 97.3% — baseline ran, but no comparable score was available; uplift unavailable | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 92.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 52.0% → 100.0% (+48.0 points) | 64.0% → 96.0% (+32.0 points) | -| Discoverability | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 44.0% → 96.0% (+52.0 points) | 42.0% → 84.0% (+42.0 points) | -| Efficiency | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 52.0% → 100.0% (+48.0 points) | 72.0% → 96.0% (+24.0 points) | +| Discoverability | 97.5% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 52.0% → 86.0% (+34.0 points) | 46.0% → 76.0% (+30.0 points) | +| Efficiency | 98.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.0% → 96.4% (-1.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 96.3% (-1.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 88.4% → 92.2% (+3.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 88.5% → 92.1% (+3.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 610,417 | 616,054 | -5,637 | -0.92% | skill 5/5; base 5/5 | -| claude-code | kpop-01 | 154,134 | 145,524 | +8,610 | +5.92% | skill 1/1; base 1/1 | -| claude-code | kpop-02 | 153,390 | 182,068 | -28,678 | -15.75% | skill 1/1; base 1/1 | -| claude-code | kpop-03 | 113,190 | 107,162 | +6,028 | +5.63% | skill 1/1; base 1/1 | -| claude-code | kpop-04 | 154,047 | 145,611 | +8,436 | +5.79% | skill 1/1; base 1/1 | -| claude-code | kpop-05 | 35,656 | 35,689 | -33 | -0.09% | skill 1/1; base 1/1 | -| codex | All cases | 209,499 | 284,139 | -74,640 | -26.27% | skill 5/5; base 5/5 | -| codex | kpop-01 | 60,262 | 66,089 | -5,827 | -8.82% | skill 1/1; base 1/1 | -| codex | kpop-02 | 43,725 | 56,299 | -12,574 | -22.33% | skill 1/1; base 1/1 | -| codex | kpop-03 | 43,942 | 60,209 | -16,267 | -27.02% | skill 1/1; base 1/1 | -| codex | kpop-04 | 43,867 | 83,937 | -40,070 | -47.74% | skill 1/1; base 1/1 | -| codex | kpop-05 | 17,703 | 17,605 | +98 | +0.56% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 819,916 | 900,193 | -80,277 | -8.92% | skill 10/10; base 10/10 | +| claude-code | All cases | 612,662 | 615,210 | -2,548 | -0.41% | skill 5/5; base 5/5 | +| claude-code | kpop-01 | 154,125 | 110,465 | +43,660 | +39.52% | skill 1/1; base 1/1 | +| claude-code | kpop-02 | 153,436 | 144,504 | +8,932 | +6.18% | skill 1/1; base 1/1 | +| claude-code | kpop-03 | 154,844 | 142,115 | +12,729 | +8.96% | skill 1/1; base 1/1 | +| claude-code | kpop-04 | 114,529 | 182,472 | -67,943 | -37.23% | skill 1/1; base 1/1 | +| claude-code | kpop-05 | 35,728 | 35,654 | +74 | +0.21% | skill 1/1; base 1/1 | +| codex | All cases | 243,157 | 280,860 | -37,703 | -13.42% | skill 5/5; base 5/5 | +| codex | kpop-01 | 60,533 | 65,793 | -5,260 | -7.99% | skill 1/1; base 1/1 | +| codex | kpop-02 | 60,696 | 60,548 | +148 | +0.24% | skill 1/1; base 1/1 | +| codex | kpop-03 | 60,404 | 59,522 | +882 | +1.48% | skill 1/1; base 1/1 | +| codex | kpop-04 | 43,726 | 77,426 | -33,700 | -43.53% | skill 1/1; base 1/1 | +| codex | kpop-05 | 17,798 | 17,571 | +227 | +1.29% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 855,819 | 896,070 | -40,251 | -4.49% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-custom-populations/skill-card.md b/skills/kumo-custom-populations/skill-card.md index ca038fe3d..3c4f731a9 100644 --- a/skills/kumo-custom-populations/skill-card.md +++ b/skills/kumo-custom-populations/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill to train Kumo models on caller-supplied labels, weights and splits, or to score exact entity/anchor rows from Snowflake source tables, when generated populations do not represent the requested rows.
+Developers and data scientists using Kumo Engine use this skill to train models on their own Snowflake labels, weights and manual splits, or to score exact entity/anchor rows, when generated populations do not represent the requested rows.
### Deployment Geography for Use:
Global
@@ -31,9 +31,9 @@ Mitigation: Review and scan skill before deployment.
## Skill Output:
**Output Type(s):** [Analysis, Code, Configuration instructions]
-**Output Format:** [Markdown with inline Python code blocks]
+**Output Format:** [Markdown with inline Python and SQL code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Presents a plan and stops for user confirmation before generating code]
+**Other Properties Related to Output:** [Presents a plan and stops for user confirmation before any code; read-only source queries only]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -42,7 +42,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod; each task was also attempted without the skill as a baseline.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod (`k8s-sandbox`); evaluator version `1.6.10`, evaluated 2026-11-19. Baseline is the same task attempted without the skill.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -74,12 +74,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 97.3% — baseline ran, but no comparable score was available; uplift unavailable | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 92.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 52.0% → 100.0% (+48.0 points) | 64.0% → 96.0% (+32.0 points) | -| Discoverability | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 44.0% → 96.0% (+52.0 points) | 42.0% → 84.0% (+42.0 points) | -| Efficiency | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 52.0% → 100.0% (+48.0 points) | 72.0% → 96.0% (+24.0 points) | +| Discoverability | 97.5% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 52.0% → 86.0% (+34.0 points) | 46.0% → 76.0% (+30.0 points) | +| Efficiency | 98.1% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -94,12 +94,20 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.0% → 96.4% (-1.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 96.3% (-1.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 88.4% → 92.2% (+3.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 88.5% → 92.1% (+3.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 11 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | ## Skill Version(s):
-0.1.0 (source: pyproject.toml)
+0.1.0 (source: pyproject.toml); git SHA 2bbedbe6, committed 2026-11-18
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-custom-populations/skill.oms.sig b/skills/kumo-custom-populations/skill.oms.sig index 4da83496c..f4cf52ed6 100644 --- a/skills/kumo-custom-populations/skill.oms.sig +++ b/skills/kumo-custom-populations/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1jdXN0b20tcG9wdWxhdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYWY1MmIyOWI1NjIzMzM1YWY2NDNlNWM0MzI0ZDA3NTAxMTFkOWEzYjI1NDZhOTY1ZmUwOWJlYjVkNmExZjAzYyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIwMTQ0MGIyOGNiZDA5YzE1N2RiNjgwODgyNzJlMTFmN2ZhYzkwZjc3ZTQ4NTcxYzFkNDU5NTEwNjI4ZWY0YjY1IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjYyMjgyZmQzNjQyYjQzMzM2NjU2MmJhNTlhNTNiYTViOTMwMjU3ZTczZTNhZDU4Yjk2ZDY4YzI3MDBmMzk3NTgiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICI3MDVmNTZjMGQ3MmYzNWJlYTlkMjEzMGU1NjJlNTBmOTlhMTNiOTExNmQ5ZjlkNGQzZDdlNWY5N2RhYzlkN2Q4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcG9wdWxhdGlvbnMubWQiLAogICAgICAgICJkaWdlc3QiOiAiN2YyMmZiNDM3ZTBjZjAwZDFjMGNlZWU2MmEwYmI3ZWYwYWYzNTQ4ZDM0MjI5ZWZmZDFjMDU1Y2I5ZTJhZDI3YSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjUyYmZjNWZjMzJhZmViZjM2ZTdmMjI1ODBiNmVlZDg1MDA0OTI3YTZmYTlhNTE3ZjY1Y2ViNGI1YWI0MzM0MmQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMET9OMbkC3KjVYlwQ+HUyK+XVTp8Qo6Y8g8VvlXhajtcufb1yQpuX2D1bz7oFlAyiwIxALGZe9eIF0zTTB41++Bj3QpLp1w1iVa6FzQ5OzMzRGhDfEiJtF7TwOYk5rvqk2mg9g==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1jdXN0b20tcG9wdWxhdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYzIxMDc4ZDY1OGYxOTE5NjI5MDI0Y2Q3YzQ4YzU4ZTc1YTc1NGE3N2IxOTgwNzZjMjdkYTUwODEyNWUxNjI4MCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkZjMwNzNiY2M3NmE5NzE3NDk1N2U3N2I0NjI5ZGU0ZGUwN2E0NWE4ZWY3MGViNzBiODUzZjVhNmU1ZWM1NmFkIiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2MjI4MmZkMzY0MmI0MzMzNjY1NjJiYTU5YTUzYmE1YjkzMDI1N2U3M2UzYWQ1OGI5NmQ2OGMyNzAwZjM5NzU4IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjcwNWY1NmMwZDcyZjM1YmVhOWQyMTMwZTU2MmU1MGY5OWExM2I5MTE2ZDlmOWQ0ZDNkN2U1Zjk3ZGFjOWQ3ZDgiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3ZjIyZmI0MzdlMGNmMDBkMWMwY2VlZTYyYTBiYjdlZjBhZjM1NDhkMzQyMjllZmZkMWMwNTVjYjllMmFkMjdhIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BvcHVsYXRpb25zLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNDQ2ZGUyMzcyNWJlZDA3ZGI2MWZiMjRjNGRkMjhkMzQ2ZGFlYTY4NjM3M2EzNjY4N2IzZjBkNDkzNWYzNDYzZCIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMC5xb67Gq4//X4nF7jlMM1+K9BEyjHxMVUAM/0p29tbleVTwudOzYCcb1RM6hUZKBgIxAI+0DzXiKVu/uY/giDHarEANpG3wDL0NsoDoWoIVcTkP/Bz5bioBRVKpCRqp2Zsglw==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-data-science/BENCHMARK.md b/skills/kumo-data-science/BENCHMARK.md index 48d198777..c4ddbcb06 100644 --- a/skills/kumo-data-science/BENCHMARK.md +++ b/skills/kumo-data-science/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 97.6% — baseline ran, but no comparable score was available; uplift unavailable | 95.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 93.8% → 100.0% (+6.2 points) | -| Correctness | 47.5% → 100.0% (+52.5 points) | 45.0% → 100.0% (+55.0 points) | -| Discoverability | 98.6% — baseline ran, but no comparable score was available; uplift unavailable | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 34.7% → 93.8% (+59.1 points) | 36.0% → 92.2% (+56.2 points) | -| Efficiency | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | 95.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 75.0% → 100.0% (+25.0 points) | +| Correctness | 52.5% → 97.5% (+45.0 points) | 50.0% → 95.0% (+45.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 39.3% → 100.0% (+60.7 points) | 37.8% → 86.6% (+48.8 points) | +| Efficiency | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | 95.3% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 96.9% → 91.8% (-5.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.0% → 92.3% (-4.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 79.9% → 90.5% (+10.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 82.0% → 90.7% (+8.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,25 +80,25 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,532,548 | 1,046,711 | +485,837 | +46.42% | skill 8/8; base 8/8 | -| claude-code | kds-01 | 179,990 | 142,821 | +37,169 | +26.02% | skill 1/1; base 1/1 | -| claude-code | kds-02 | 242,679 | 161,366 | +81,313 | +50.39% | skill 1/1; base 1/1 | -| claude-code | kds-03 | 236,518 | 276,359 | -39,841 | -14.42% | skill 1/1; base 1/1 | -| claude-code | kds-04 | 267,189 | 144,990 | +122,199 | +84.28% | skill 1/1; base 1/1 | -| claude-code | kds-05 | 170,568 | 70,704 | +99,864 | +141.24% | skill 1/1; base 1/1 | -| claude-code | kds-06 | 217,878 | 107,637 | +110,241 | +102.42% | skill 1/1; base 1/1 | -| claude-code | kds-07 | 181,830 | 107,195 | +74,635 | +69.63% | skill 1/1; base 1/1 | -| claude-code | kds-08 | 35,896 | 35,639 | +257 | +0.72% | skill 1/1; base 1/1 | -| codex | All cases | 572,619 | 2,940,672 | -2,368,053 | -80.53% | skill 8/8; base 8/8 | -| codex | kds-01 | 69,610 | 49,453 | +20,157 | +40.76% | skill 1/1; base 1/1 | -| codex | kds-02 | 97,805 | 1,448,814 | -1,351,009 | -93.25% | skill 1/1; base 1/1 | -| codex | kds-03 | 96,860 | 1,286,574 | -1,189,714 | -92.47% | skill 1/1; base 1/1 | -| codex | kds-04 | 93,764 | 13,570 | +80,194 | +590.97% | skill 1/1; base 1/1 | -| codex | kds-05 | 84,347 | 30,117 | +54,230 | +180.06% | skill 1/1; base 1/1 | -| codex | kds-06 | 66,995 | 41,092 | +25,903 | +63.04% | skill 1/1; base 1/1 | -| codex | kds-07 | 49,516 | 57,606 | -8,090 | -14.04% | skill 1/1; base 1/1 | -| codex | kds-08 | 13,722 | 13,446 | +276 | +2.05% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 2,105,167 | 3,987,383 | -1,882,216 | -47.20% | skill 16/16; base 16/16 | +| claude-code | All cases | 1,546,761 | 961,349 | +585,412 | +60.89% | skill 8/8; base 8/8 | +| claude-code | kds-01 | 225,288 | 143,106 | +82,182 | +57.43% | skill 1/1; base 1/1 | +| claude-code | kds-02 | 184,106 | 161,500 | +22,606 | +14.00% | skill 1/1; base 1/1 | +| claude-code | kds-03 | 226,123 | 190,667 | +35,456 | +18.60% | skill 1/1; base 1/1 | +| claude-code | kds-04 | 267,817 | 145,317 | +122,500 | +84.30% | skill 1/1; base 1/1 | +| claude-code | kds-05 | 217,017 | 70,650 | +146,367 | +207.17% | skill 1/1; base 1/1 | +| claude-code | kds-06 | 216,857 | 107,418 | +109,439 | +101.88% | skill 1/1; base 1/1 | +| claude-code | kds-07 | 173,669 | 107,032 | +66,637 | +62.26% | skill 1/1; base 1/1 | +| claude-code | kds-08 | 35,884 | 35,659 | +225 | +0.63% | skill 1/1; base 1/1 | +| codex | All cases | 510,192 | 3,729,493 | -3,219,301 | -86.32% | skill 8/8; base 8/8 | +| codex | kds-01 | 70,556 | 45,022 | +25,534 | +56.71% | skill 1/1; base 1/1 | +| codex | kds-02 | 96,732 | 2,075,477 | -1,978,745 | -95.34% | skill 1/1; base 1/1 | +| codex | kds-03 | 96,681 | 1,411,737 | -1,315,056 | -93.15% | skill 1/1; base 1/1 | +| codex | kds-04 | 49,701 | 41,202 | +8,499 | +20.63% | skill 1/1; base 1/1 | +| codex | kds-05 | 67,801 | 30,289 | +37,512 | +123.85% | skill 1/1; base 1/1 | +| codex | kds-06 | 65,684 | 54,786 | +10,898 | +19.89% | skill 1/1; base 1/1 | +| codex | kds-07 | 49,385 | 57,575 | -8,190 | -14.22% | skill 1/1; base 1/1 | +| codex | kds-08 | 13,652 | 13,405 | +247 | +1.84% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,056,953 | 4,690,842 | -2,633,889 | -56.15% | skill 16/16; base 16/16 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -122,13 +122,13 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c "## Canonical good example (`train`)" in accept/intake_contract.md (lines 137-148) vs "## Input" in validate/examples/good_request.md (lines 3-12) vs "## Node 1a — Required fields: PASS" in validate/examples/good_request.md (lines 13-19) (`accept/intake_contract.md:137`) -- **HIGH** DUPLICATE/duplicate: Duplicate content found across validate/atomic_question_check.md and validate/examples/bad_request.md: - "### Fail 1 — Multiple targets bundled" in validate/atomic_question_check.md (lines 42-50) - vs "## 6. Bundled goals → REJECT" in validate/examples/bad_request.md (lines 64-68) (`validate/atomic_question_check.md:42`) - **HIGH** DUPLICATE/duplicate: Duplicate content found within accept/batch_inference.md: "## Node 1a — fields" in accept/batch_inference.md (lines 7-29) vs "### `batch_inference` — required fields (ask if missing)" in accept/batch_inference.md (lines 30-42) vs "### `batch_inference` — resolved, defaulted, or inferred (never ask)" in accept/batch_inference.md (lines 43-54) (`accept/batch_inference.md:7`) +- **HIGH** DUPLICATE/duplicate: Duplicate content found across validate/atomic_question_check.md and validate/examples/bad_request.md: + "### Fail 1 — Multiple targets bundled" in validate/atomic_question_check.md (lines 42-50) + vs "## 6. Bundled goals → REJECT" in validate/examples/bad_request.md (lines 64-68) (`validate/atomic_question_check.md:42`) - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/kumo-data-science/SKILL.md`) - 25 additional finding(s) are available in the full evaluation artifacts. diff --git a/skills/kumo-data-science/skill-card.md b/skills/kumo-data-science/skill-card.md index 4f25af972..c8ede2982 100644 --- a/skills/kumo-data-science/skill-card.md +++ b/skills/kumo-data-science/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Routes KumoAI graph and predictive-modeling requests on relational data in a warehouse, an object store or local files, and for a new model confirms the data, outcome, time window, inferred tables and task through a gated intake before generating one train-then-predict notebook or script, which it runs only when asked.
+Routes KumoAI requests on relational data in a warehouse, object store, or local files: it gates a new model through multi-turn intake (data, outcome, time window, then inferred graph and task confirmation) and generates one notebook or script that trains then predicts, runs it only on explicit request, and routes graph inspection, explanations, diagnostics, persistence, custom populations, tuning, and Kumo job requests to their workflows.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers, data scientists and ML engineers use this skill to turn plain-language predictive requests (churn, propensity, fraud, recommendation, forecasting, link prediction) on relational data into a confirmed KumoAI task and graph and one runnable notebook or script, and to route graph inspection, explanation, diagnostics, persistence, custom-population, tuning and job requests to their dedicated skills.
+Developers, data scientists, and ML engineers use this skill to have an agent turn a plain-language predictive request (churn, propensity, fraud, recommendation, forecasting, link prediction) into a confirmed KumoAI task and graph and a runnable training or batch-inference notebook or script, and to route follow-up model and job requests.
### Deployment Geography for Use:
Global
@@ -25,23 +25,23 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [Intake contract](accept/intake_contract.md)
-- [Preflight and graph gate](accept/preflight.md)
-- [Batch inference](accept/batch_inference.md)
-- [Semantic views](accept/semantic_view.md)
-- [Atomic question check](validate/atomic_question_check.md)
-- [Generic host adapter](hosts/generic/adapter.md)
-- [Generic host runtime](hosts/generic/runtime.md)
- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
- [kumo-engine on PyPI](https://pypi.org/project/kumo-engine/)
- [NVIDIA/kumo-engine#134 (graph visualization)](https://github.com/NVIDIA/kumo-engine/pull/134)
+- [accept/intake_contract.md](accept/intake_contract.md)
+- [accept/preflight.md](accept/preflight.md)
+- [accept/semantic_view.md](accept/semantic_view.md)
+- [accept/batch_inference.md](accept/batch_inference.md)
+- [validate/atomic_question_check.md](validate/atomic_question_check.md)
+- [hosts/generic/adapter.md](hosts/generic/adapter.md)
+- [hosts/generic/runtime.md](hosts/generic/runtime.md)
## Skill Output:
**Output Type(s):** [Analysis, Code, Files]
-**Output Format:** [Markdown replies (graph and task proposals, ask-backs) and one generated Jupyter notebook or Python script using the kumo.engine API]
+**Output Format:** [Markdown intake replies (ask-back, graph proposal, task restatement) and one generated Jupyter notebook or Python script]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Generates code only after the user confirms the graph and task; runs the artifact only on explicit request]
+**Other Properties Related to Output:** [Generates no Kumo SDK code before the task is confirmed; generated artifacts run only on explicit user request]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -50,7 +50,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-8 evaluation tasks (7 positive, 1 negative), 1 attempt per task, each run in an isolated sandbox pod (`k8s-sandbox`), compared against a no-skill baseline. Evaluator version `1.6.10`; evaluated 2026-11-19.
+8 evaluation tasks (7 positive, 1 negative), 1 attempt per task, each attempt in an isolated sandbox pod (`k8s-sandbox`), compared against a no-skill baseline. Evaluator version `1.6.10`, evaluation date 2026-11-19; dataset digest `sha256:f72dc6983fe9c6ff4f46b5d40bebd4ceb50356c04e83ebc88dc2cd383486d627` (skill-evaluator-dataset-snapshot/1).
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -61,32 +61,36 @@ Reported benchmark dimensions:
- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Final-answer correctness against the reference answer.
-- `goal_accuracy`: Whether the user's goal was achieved.
-- `behavior_check`: Whether the expected workflow behavior was followed.
-- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + ### Results at a Glance | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 97.6% — baseline ran, but no comparable score was available; uplift unavailable | 95.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 93.8% → 100.0% (+6.2 points) | -| Correctness | 47.5% → 100.0% (+52.5 points) | 45.0% → 100.0% (+55.0 points) | -| Discoverability | 98.6% — baseline ran, but no comparable score was available; uplift unavailable | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 34.7% → 93.8% (+59.1 points) | 36.0% → 92.2% (+56.2 points) | -| Efficiency | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | 95.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 75.0% → 100.0% (+25.0 points) | +| Correctness | 52.5% → 97.5% (+45.0 points) | 50.0% → 95.0% (+45.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 39.3% → 100.0% (+60.7 points) | 37.8% → 86.6% (+48.8 points) | +| Efficiency | 96.1% — baseline ran, but no comparable score was available; uplift unavailable | 95.3% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. -Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. - A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. #### Comparison details @@ -96,9 +100,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 96.9% → 91.8% (-5.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.0% → 92.3% (-4.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 79.9% → 90.5% (+10.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 82.0% → 90.7% (+8.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -108,12 +112,8 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Tier 2 | Semantic deduplication | **PASSED WITH OBSERVATIONS** | 2 validator(s); 4 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 8 task(s) | -**Overall verdict: PASS — Recommended for publication** - -**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. - ## Skill Version(s):
-0.1.0 (source: pyproject.toml); skill build marker 2026-11-18-7218a192-dirty (source: SKILL.md); git bfbf737f (committed 2026-11-18)
+0.1.0 (source: pyproject.toml); SKILL.md build marker 2026-11-19-bef5ab4e-dirty, repo commit 2bbedbe6 (committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-data-science/skill.oms.sig b/skills/kumo-data-science/skill.oms.sig index 7bbb17bc0..99d70d3dc 100644 --- a/skills/kumo-data-science/skill.oms.sig +++ b/skills/kumo-data-science/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1kYXRhLXNjaWVuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiOWM1ZDllNWMyNjY1MjhiMDYyMzBlZTkwY2MyZjNlNWM3MmQ5NzkxZGY2YzU5NzRlNzVmNmRjMjE1NGFmYjcyYiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQwNjg3NGJlMDc1MmQzZmFmMTA5YzQ5YWMwNmFhYjdjOWM3Njc4YTM3MzJhYmIzZWFhYTI3OGY1OTc4NDY5NWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiM2E2Y2RlZDQyZTQ0YzRkNTliMDhkMDNiNTMzZTg3NjFmNzhlNGMyZDhlNWMzMTU2Nzc3MDdkYjVhMWEyMzVhYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImFjY2VwdC9iYXRjaF9pbmZlcmVuY2UubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjJkNzVhYmEwZmE0ZmUxOWI4ODU2NTMyODBlMWE1MmQxM2IwN2EwZTEzM2RjNDM0ZDFiMmJkN2FiMGY5ZjZiYjgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJhY2NlcHQvaW50YWtlX2NvbnRyYWN0Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmZWE3YmJkODY1YjgzNjY0MTEwN2Y3NDAzNDgyNmIxZjU0YTE0ZDE0ZjM4YjE3ZWFjMGZkZGE4MDg5NDEyMmQxIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L3ByZWZsaWdodC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDNiNGM5MDc0MTVlYTQzNjBlMDlhNGFhMjJlOWNhOGU2OGJkMjNmOGI0YzQ3Y2NmOWQxYjk1ZDhhYTYxYmU5ZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImFjY2VwdC9zZW1hbnRpY192aWV3Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmOWYyZGY5Y2FhNGQ1NjllNDNlNGQzNGY3MDk5YjIxNTExZmFjMWFiMzllZDYyMjczMjQ4ZmI2MTFiOWNiZDNlIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjY0MDRkOWNiMTNkMDA5M2I2NzAxZGZkNGIxOTMzNzEwM2Q4ZDNiODYwNzVlMTExMzkzNmQzYjQ0ZTk5Y2NhOSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImhvc3RzL2dlbmVyaWMvYWRhcHRlci5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYzZhMzY4NDU0ZmE5YWRlZWJhMGJkZGE5OGJkM2QyMTRmMmI5MmE1ZjEyODM1Mjc0NWMzZDIyZWI5MDdiNWI2YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImhvc3RzL2dlbmVyaWMvcnVudGltZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjZjNmM5NDFiMWQwZTdlYTA5OWJjNTRjNzBmODlkZWYwZGM4NDI5YmNjZDQzYjY1MGQ5NmMyNThjYzhmZWNmYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjU3YTEyMWU2NjFiNDlhYTIzYmRjMWI2ZjcwNTVkMzVlNmQ1NmExYzBjOGNjMzQ4MTdkMTc5NTIwNjBlMzQyMzgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJ2YWxpZGF0ZS9hdG9taWNfcXVlc3Rpb25fY2hlY2subWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjY1MDZlNjYxYmMxMWMyNWViNjZhN2I3YzM1YTM0MDMxMzk1N2QzMjRiYTM4NjdlOTUwODZiMTE0ZTkyZDIwNGEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJ2YWxpZGF0ZS9leGFtcGxlcy9iYWRfcmVxdWVzdC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2JkMzkzNGUxYjljMjJhNGMzOGIzN2JjYjhjNTk3MjkyZGRlZTgyMjA2MWQ5ZmVlZWU2NTdiZDkwNTAzNWNkNCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2V4YW1wbGVzL2dvb2RfcmVxdWVzdC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiODkyZDBmYmViNDJhODAyZDMzNmZmYjIzMDlkNWU5OTliMzhjZmI0ODdkMzY5YzM4OTI5NDlmM2NjMWJlZDhlNiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDa7oeoHp4Wc28ILUajCblNJ7/7/2P51d+mllAZd/qWDZ8OZ3at5+VJnNTbPuU1IXgCMEFOFWSlEepXDqXzpePUSFNAeQFQuZ33WLbI0wmDebMbP55QtXZvGkyOr8h45H08LQ==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1kYXRhLXNjaWVuY2UiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiOWIxY2E1NmRlMThiYjkyNWQzMzI1ZGM0YjU1M2MxOTcxMzczZDhhMWQxMzE1NDNhNmY3YTBkMjlkY2ZiNWYxNiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjRkNTdiNGMxNDI0NjJhYTUxMGIyMzhlY2QxOTUwNDVjZDY0YjQ5OWMzMWQ0YzRlNTRiZDhlMTVmYmMxNzFmNjciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDE5NjY2OTgzMGU3ZjEwOTU5Y2MyNWQyZDcxY2NlMDhmM2I4Njg1ZDY5ZDg3NzJhYTIzZDk1MDM1NzNiNTFjNyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImFjY2VwdC9iYXRjaF9pbmZlcmVuY2UubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjJkNzVhYmEwZmE0ZmUxOWI4ODU2NTMyODBlMWE1MmQxM2IwN2EwZTEzM2RjNDM0ZDFiMmJkN2FiMGY5ZjZiYjgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJhY2NlcHQvaW50YWtlX2NvbnRyYWN0Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmZWE3YmJkODY1YjgzNjY0MTEwN2Y3NDAzNDgyNmIxZjU0YTE0ZDE0ZjM4YjE3ZWFjMGZkZGE4MDg5NDEyMmQxIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiYWNjZXB0L3ByZWZsaWdodC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDNiNGM5MDc0MTVlYTQzNjBlMDlhNGFhMjJlOWNhOGU2OGJkMjNmOGI0YzQ3Y2NmOWQxYjk1ZDhhYTYxYmU5ZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImFjY2VwdC9zZW1hbnRpY192aWV3Lm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmOWYyZGY5Y2FhNGQ1NjllNDNlNGQzNGY3MDk5YjIxNTExZmFjMWFiMzllZDYyMjczMjQ4ZmI2MTFiOWNiZDNlIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjY0MDRkOWNiMTNkMDA5M2I2NzAxZGZkNGIxOTMzNzEwM2Q4ZDNiODYwNzVlMTExMzkzNmQzYjQ0ZTk5Y2NhOSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImhvc3RzL2dlbmVyaWMvYWRhcHRlci5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYzZhMzY4NDU0ZmE5YWRlZWJhMGJkZGE5OGJkM2QyMTRmMmI5MmE1ZjEyODM1Mjc0NWMzZDIyZWI5MDdiNWI2YiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImhvc3RzL2dlbmVyaWMvcnVudGltZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMTRlYmQ2NTYzYmZmOTA2MDkxNmIwYjFjOWU0NzU4ODdkYjRiZWZkZDQ1OTUzMGIwMjk2N2JmNjYxYTc3OGRhYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImEwODkwZWViYTAyNTAyOTkyZTJlODAxOGYxMmQ2ODM5MzM3MDI3YjVlOWNkNDFhZTE4MTcxMThjNTM4YTMxYjciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJ2YWxpZGF0ZS9hdG9taWNfcXVlc3Rpb25fY2hlY2subWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjY1MDZlNjYxYmMxMWMyNWViNjZhN2I3YzM1YTM0MDMxMzk1N2QzMjRiYTM4NjdlOTUwODZiMTE0ZTkyZDIwNGEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJ2YWxpZGF0ZS9leGFtcGxlcy9iYWRfcmVxdWVzdC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiN2JkMzkzNGUxYjljMjJhNGMzOGIzN2JjYjhjNTk3MjkyZGRlZTgyMjA2MWQ5ZmVlZWU2NTdiZDkwNTAzNWNkNCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInZhbGlkYXRlL2V4YW1wbGVzL2dvb2RfcmVxdWVzdC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiODkyZDBmYmViNDJhODAyZDMzNmZmYjIzMDlkNWU5OTliMzhjZmI0ODdkMzY5YzM4OTI5NDlmM2NjMWJlZDhlNiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMFOSMgjK6UG8hThOGLo7CSodgs29b6oB7raaeISyvC4KlwhRA6umvY084u2yTc1JGwIwFP67X9MHcMnKvZA+0Qv2x/fIHQvgnPDtvinl6s+E2+yhLAckDVrEH0GzC/Kxk7yx","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-graph-visualization/BENCHMARK.md b/skills/kumo-graph-visualization/BENCHMARK.md index 194682097..4f7f3d007 100644 --- a/skills/kumo-graph-visualization/BENCHMARK.md +++ b/skills/kumo-graph-visualization/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | 78.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | 78.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 72.0% (+44.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 30.8% → 50.0% (+19.2 points) | 32.5% → 37.5% (+5.0 points) | -| Efficiency | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 64.0% (+36.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 30.8% → 50.0% (+19.2 points) | 30.0% → 37.5% (+7.5 points) | +| Efficiency | 92.6% — baseline ran, but no comparable score was available; uplift unavailable | 96.5% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 97.3% (-1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.9% → 97.7% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 92.3% → 92.0% (-0.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 91.8% → 93.0% (+1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 880,796 | 786,371 | +94,425 | +12.01% | skill 5/5; base 5/5 | -| claude-code | kviz-01 | 145,931 | 141,553 | +4,378 | +3.09% | skill 1/1; base 1/1 | -| claude-code | kviz-02 | 110,065 | 107,381 | +2,684 | +2.50% | skill 1/1; base 1/1 | -| claude-code | kviz-03 | 147,135 | 143,223 | +3,912 | +2.73% | skill 1/1; base 1/1 | -| claude-code | kviz-04 | 147,314 | 142,938 | +4,376 | +3.06% | skill 1/1; base 1/1 | -| claude-code | kviz-05 | 330,351 | 251,276 | +79,075 | +31.47% | skill 1/1; base 1/1 | -| codex | All cases | 239,429 | 244,628 | -5,199 | -2.13% | skill 5/5; base 5/5 | -| codex | kviz-01 | 57,428 | 68,472 | -11,044 | -16.13% | skill 1/1; base 1/1 | -| codex | kviz-02 | 27,482 | 41,450 | -13,968 | -33.70% | skill 1/1; base 1/1 | -| codex | kviz-03 | 57,177 | 40,594 | +16,583 | +40.85% | skill 1/1; base 1/1 | -| codex | kviz-04 | 54,891 | 53,225 | +1,666 | +3.13% | skill 1/1; base 1/1 | -| codex | kviz-05 | 42,451 | 40,887 | +1,564 | +3.83% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,120,225 | 1,030,999 | +89,226 | +8.65% | skill 10/10; base 10/10 | +| claude-code | All cases | 802,363 | 827,591 | -25,228 | -3.05% | skill 5/5; base 5/5 | +| claude-code | kviz-01 | 183,641 | 141,611 | +42,030 | +29.68% | skill 1/1; base 1/1 | +| claude-code | kviz-02 | 109,867 | 106,924 | +2,943 | +2.75% | skill 1/1; base 1/1 | +| claude-code | kviz-03 | 184,467 | 142,840 | +41,627 | +29.14% | skill 1/1; base 1/1 | +| claude-code | kviz-04 | 110,060 | 143,467 | -33,407 | -23.29% | skill 1/1; base 1/1 | +| claude-code | kviz-05 | 214,328 | 292,749 | -78,421 | -26.79% | skill 1/1; base 1/1 | +| codex | All cases | 227,272 | 267,228 | -39,956 | -14.95% | skill 5/5; base 5/5 | +| codex | kviz-01 | 42,441 | 68,494 | -26,053 | -38.04% | skill 1/1; base 1/1 | +| codex | kviz-02 | 27,464 | 73,962 | -46,498 | -62.87% | skill 1/1; base 1/1 | +| codex | kviz-03 | 72,767 | 27,034 | +45,733 | +169.17% | skill 1/1; base 1/1 | +| codex | kviz-04 | 42,085 | 57,335 | -15,250 | -26.60% | skill 1/1; base 1/1 | +| codex | kviz-05 | 42,515 | 40,403 | +2,112 | +5.23% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,029,635 | 1,094,819 | -65,184 | -5.95% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-graph-visualization/skill-card.md b/skills/kumo-graph-visualization/skill-card.md index 2410ffe8c..8f77061be 100644 --- a/skills/kumo-graph-visualization/skill-card.md +++ b/skills/kumo-graph-visualization/skill-card.md @@ -72,38 +72,32 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 88.0% — baseline ran, but no comparable score was available; uplift unavailable | 78.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | 78.4% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 72.0% (+44.0 points) | -| Discoverability | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 30.8% → 50.0% (+19.2 points) | 32.5% → 37.5% (+5.0 points) | -| Efficiency | 91.4% — baseline ran, but no comparable score was available; uplift unavailable | 88.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 36.0% → 100.0% (+64.0 points) | 28.0% → 64.0% (+36.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 30.8% → 50.0% (+19.2 points) | 30.0% → 37.5% (+7.5 points) | +| Efficiency | 92.6% — baseline ran, but no comparable score was available; uplift unavailable | 96.5% — baseline ran, but no comparable score was available; uplift unavailable | -Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. -### Comparison details +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 97.3% (-1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.9% → 97.7% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 92.3% → 92.0% (-0.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | - -### Tier Status - -| Tier | Purpose | Status | Evidence | -|---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 10 finding(s) | -| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | - -Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. Overall verdict: PASS only when every configured dimension passes for at least one supported agent. +| codex | Efficiency | Token-only comparison | 91.8% → 93.0% (+1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
-0.1.0 (source: pyproject.toml)
+0.1.0 (source: pyproject.toml; git SHA 2bbedbe6, committed 2026-11-18)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumo-graph-visualization/skill.oms.sig b/skills/kumo-graph-visualization/skill.oms.sig index b7e9eb0df..accfcd838 100644 --- a/skills/kumo-graph-visualization/skill.oms.sig +++ b/skills/kumo-graph-visualization/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1ncmFwaC12aXN1YWxpemF0aW9uIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImQ0ODFkNmNkNTlkNmQ2ODU5N2I3MWRhYjgwNDQwNmUwOWNmM2NjMjhhMTdhMDk4ZjZiMjZhMjJiNjhjNjQ5YTIiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0KICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIyY2FkZWU2ZDcyMjU1MGE2Zjg0OWE1MTNhODVhN2JhMzJmYjU1ODc2NDQ5YzcxN2FmZjY5N2JhYWEyMjE5MzI3IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjk1NjNjMmQ1ZDRkMzJiYWIwMjhhNGY5NmJhMTVlNDQ1ZDdhZWMyMmYxM2Y0ZGJmNDQ3NjVlY2Y1M2IwYzhmZjciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3NzRhMjc4YTY2OTBmZTBiZjcxMDAxZDc2MDk1ZDM2NmQxMGE4MDNjOGE2Njg2NWRkZjM0YTMyZTc1ZTZiOTliIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiODA3MjcxYmY3M2VmYTE3OWNmNThiOWExNGM0NDI4ZDU3NTU2YTJhOTFmZTQ4Y2E3OTk5ZmJjZjJhYTkwMzFhMCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMBHHd1ds0NngNnG3J/24Y2GBzGkZsMTDxxqT4oh7p6HxDFGorZkSml5JbK4gRyc5MAIwGSqg5GcZF3lIV4eiIpIVgGJV0EeCZfzQQ3qfluGXQ+jCTnMx6qVARkEJwWjAgL6B","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1ncmFwaC12aXN1YWxpemF0aW9uIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjM4ZjYxNjk1NGVjM2VmZGYzMmU0ZmU3ZWVhNzU1Yzg3ZGI2ZWU2ZDcwYzQ0M2EwMjczY2FmNWU4Mzg3Y2JkNTMiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogImZhZWFiNDA5ZWQyYTUyNDlkZmE5YzI5M2EyYTQ3MDg4M2QyZjdhNThmMzA0OGUwYjZmYTZmNTE3OWRlNDAxZjciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiOTU2M2MyZDVkNGQzMmJhYjAyOGE0Zjk2YmExNWU0NDVkN2FlYzIyZjEzZjRkYmY0NDc2NWVjZjUzYjBjOGZmNyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjc3NGEyNzhhNjY5MGZlMGJmNzEwMDFkNzYwOTVkMzY2ZDEwYTgwM2M4YTY2ODY1ZGRmMzRhMzJlNzVlNmI5OWIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI2YTM0MTdhMDYxOGRjOWEwMGRlYmM2MzRjZjkyZGI3Y2I4MjNkZDczNjNlMzI0MTVjMzFiMGYyMDY5ODllMzY3IgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIKICAgICAgXSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQC0jWH+Sz0pwCrFezcVg17cPs1x59DOHH/yZCSymHpYyqpT3q3nJNn+/FC8MTC5HzECMEQLZGPZl58vngtElXGK8dR/ZipjwJwrA5ERxBJoxhbZ85qbKC3QeWiR1QanMXwrTg==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-explanations/BENCHMARK.md b/skills/kumo-model-explanations/BENCHMARK.md index 0ad6aa830..ec640e156 100644 --- a/skills/kumo-model-explanations/BENCHMARK.md +++ b/skills/kumo-model-explanations/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | 87.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 43.3% → 100.0% (+56.7 points) | 40.0% → 93.3% (+53.3 points) | +| Correctness | 46.7% → 100.0% (+53.3 points) | 56.7% → 90.0% (+33.3 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 25.0% → 64.6% (+39.6 points) | 22.9% → 56.3% (+33.4 points) | -| Efficiency | 97.9% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 25.0% → 64.6% (+39.6 points) | 41.7% → 66.0% (+24.3 points) | +| Efficiency | 98.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.1% → 95.7% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 95.9% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.6% → 91.6% (+1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.5% → 91.7% (+1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,21 +80,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 886,610 | 896,669 | -10,059 | -1.12% | skill 6/6; base 6/6 | -| claude-code | kexpl-01 | 116,441 | 142,323 | -25,882 | -18.19% | skill 1/1; base 1/1 | -| claude-code | kexpl-02 | 158,908 | 144,073 | +14,835 | +10.30% | skill 1/1; base 1/1 | -| claude-code | kexpl-03 | 116,786 | 143,330 | -26,544 | -18.52% | skill 1/1; base 1/1 | -| claude-code | kexpl-04 | 159,748 | 143,054 | +16,694 | +11.67% | skill 1/1; base 1/1 | -| claude-code | kexpl-05 | 226,319 | 178,371 | +47,948 | +26.88% | skill 1/1; base 1/1 | -| claude-code | kexpl-06 | 108,408 | 145,518 | -37,110 | -25.50% | skill 1/1; base 1/1 | -| codex | All cases | 416,838 | 286,190 | +130,648 | +45.65% | skill 6/6; base 6/6 | -| codex | kexpl-01 | 81,433 | 43,854 | +37,579 | +85.69% | skill 1/1; base 1/1 | -| codex | kexpl-02 | 83,133 | 41,340 | +41,793 | +101.10% | skill 1/1; base 1/1 | -| codex | kexpl-03 | 84,670 | 56,514 | +28,156 | +49.82% | skill 1/1; base 1/1 | -| codex | kexpl-04 | 81,171 | 62,562 | +18,609 | +29.74% | skill 1/1; base 1/1 | -| codex | kexpl-05 | 45,285 | 41,057 | +4,228 | +10.30% | skill 1/1; base 1/1 | -| codex | kexpl-06 | 41,146 | 40,863 | +283 | +0.69% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,303,448 | 1,182,859 | +120,589 | +10.19% | skill 12/12; base 12/12 | +| claude-code | All cases | 849,436 | 751,324 | +98,112 | +13.06% | skill 6/6; base 6/6 | +| claude-code | kexpl-01 | 158,300 | 142,249 | +16,051 | +11.28% | skill 1/1; base 1/1 | +| claude-code | kexpl-02 | 159,108 | 107,825 | +51,283 | +47.56% | skill 1/1; base 1/1 | +| claude-code | kexpl-03 | 116,513 | 180,575 | -64,062 | -35.48% | skill 1/1; base 1/1 | +| claude-code | kexpl-04 | 159,810 | 107,520 | +52,290 | +48.63% | skill 1/1; base 1/1 | +| claude-code | kexpl-05 | 147,869 | 141,899 | +5,970 | +4.21% | skill 1/1; base 1/1 | +| claude-code | kexpl-06 | 107,836 | 71,256 | +36,580 | +51.34% | skill 1/1; base 1/1 | +| codex | All cases | 414,153 | 287,331 | +126,822 | +44.14% | skill 6/6; base 6/6 | +| codex | kexpl-01 | 80,597 | 57,938 | +22,659 | +39.11% | skill 1/1; base 1/1 | +| codex | kexpl-02 | 80,794 | 40,999 | +39,795 | +97.06% | skill 1/1; base 1/1 | +| codex | kexpl-03 | 84,804 | 52,955 | +31,849 | +60.14% | skill 1/1; base 1/1 | +| codex | kexpl-04 | 81,383 | 53,824 | +27,559 | +51.20% | skill 1/1; base 1/1 | +| codex | kexpl-05 | 45,317 | 41,108 | +4,209 | +10.24% | skill 1/1; base 1/1 | +| codex | kexpl-06 | 41,258 | 40,507 | +751 | +1.85% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,263,589 | 1,038,655 | +224,934 | +21.66% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-explanations/skill-card.md b/skills/kumo-model-explanations/skill-card.md index 95025fe2b..a7198593a 100644 --- a/skills/kumo-model-explanations/skill-card.md +++ b/skills/kumo-model-explanations/skill-card.md @@ -72,12 +72,12 @@ Underlying evaluation signals used in this run:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | 87.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 43.3% → 100.0% (+56.7 points) | 40.0% → 93.3% (+53.3 points) | +| Correctness | 46.7% → 100.0% (+53.3 points) | 56.7% → 90.0% (+33.3 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 25.0% → 64.6% (+39.6 points) | 22.9% → 56.3% (+33.4 points) | -| Efficiency | 97.9% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 25.0% → 64.6% (+39.6 points) | 41.7% → 66.0% (+24.3 points) | +| Efficiency | 98.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -92,9 +92,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.1% → 95.7% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 95.9% (-2.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.6% → 91.6% (+1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.5% → 91.7% (+1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
0.1.0 (source: pyproject.toml)
diff --git a/skills/kumo-model-explanations/skill.oms.sig b/skills/kumo-model-explanations/skill.oms.sig index 2b9949b10..7ad35494e 100644 --- a/skills/kumo-model-explanations/skill.oms.sig +++ b/skills/kumo-model-explanations/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1leHBsYW5hdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNzllMDBjYjA4NjljNDE4ZDA5ZmU4YTM2ODY0OWU4NTk4ZGM3MGRhNjFiODI3MjMyNjQ1Mzg0YTBhYTcyMDE4OCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIKICAgICAgXSwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjYwNDVhY2JiMDdmOWUzZmQ2ODdlNjg2Nzk4MmMzYjIwNjE0MjUxZjdhOTkwOWRlNjIzZjdjNzc4NjIxODQ4MTAiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImNiNzRiN2RkY2RkM2FlMDZjNTJiYzg1MmNkOWM5NzllOTRiYTY0MGM2ZWNlODIyZjVmNDkyYmJmZjk3ZGFhMGMiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOTY3NGE3MWExYTFiZjA2ZTVmM2EwN2U0Y2NlZGU1N2QxMzM2NjM5ZjQ0YzIxZjAzNDNhZDcyYTdlMDcyZGIxNyIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjUwY2I2OGZhYmQ3NGM4ZTFlOTJjNGM4MzBlZTUwOWVkY2I2NjU0YjcwZjk4YzI3Y2U4ZmUzNGM4OThlYTcxN2EiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvYXJ0aWZhY3RzLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiY2JkMzhhZWRlOTg1MzRmN2Y4ZDhjNjY2NWE2MDYwMDllMDgzYWFmOWQyOTkxNDg1YWVlM2I4MTQzNTBjNTVmMiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDzHlEctDcF654TNEYk+yVRU3bfsfHEV33M6rRQye4MHF7AwcAjODovCuNexs42PzsCMQDl7wncFGbDf3b69nktSmqrEGt3AA7EKfgfkNSk5dhroI5LyM6+eUHQ4YHP3q4ZxsU=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1leHBsYW5hdGlvbnMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiN2JjYzVkMzY2MWQxMDViNjAwMWU2OWI0ZDA1NWVmOGE4YjcyMDQ4Njk3NzAwMDA5MjFkMmZhNWFkYjAyMzE2YSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjZjZmUyMzk1M2IzMDNkYjE1YmYxZjEyNjA2NDNhNzNhY2ZiMDE0YTYyYTNhNzQ1MjBhMDk2M2FkYTM0OGJmMzAiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImNiNzRiN2RkY2RkM2FlMDZjNTJiYzg1MmNkOWM5NzllOTRiYTY0MGM2ZWNlODIyZjVmNDkyYmJmZjk3ZGFhMGMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOTY3NGE3MWExYTFiZjA2ZTVmM2EwN2U0Y2NlZGU1N2QxMzM2NjM5ZjQ0YzIxZjAzNDNhZDcyYTdlMDcyZGIxNyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjUwY2I2OGZhYmQ3NGM4ZTFlOTJjNGM4MzBlZTUwOWVkY2I2NjU0YjcwZjk4YzI3Y2U4ZmUzNGM4OThlYTcxN2EiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2FydGlmYWN0cy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZGEwMTU2Mjk1ZjQ5YWM2ZjYzNGE1MDU5NzE4ZjM5NTE4ZWY0MDM0YjJjODJmYzY2NWQ4MDZjZDhkZTc1MTRjNiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDWX8ltqH4F8bdQjhghhP5BE1Uim1Fcmxun3PAroxJyt/MudF/Q2Gy+KXeUJQg3tGICMB3pJ7cp6uEfKki1YmIptOhxYYSmfY1PkcX8SER9balJzPquidLexe2tyTVZlqhemg==","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-persistence/BENCHMARK.md b/skills/kumo-model-persistence/BENCHMARK.md index 7b06e0c78..e921209b8 100644 --- a/skills/kumo-model-persistence/BENCHMARK.md +++ b/skills/kumo-model-persistence/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.1% — baseline ran, but no comparable score was available; uplift unavailable | 80.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 91.7% → 100.0% (+8.3 points) | -| Correctness | 56.7% → 96.7% (+40.0 points) | 40.0% → 83.3% (+43.3 points) | -| Discoverability | 98.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 32.5% → 73.9% (+41.4 points) | 27.2% → 47.2% (+20.0 points) | -| Efficiency | 92.1% — baseline ran, but no comparable score was available; uplift unavailable | 82.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 91.8% — baseline ran, but no comparable score was available; uplift unavailable | 84.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 60.0% → 100.0% (+40.0 points) | 50.0% → 86.7% (+36.7 points) | +| Discoverability | 99.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 31.1% → 70.6% (+39.5 points) | 27.2% → 66.1% (+38.9 points) | +| Efficiency | 89.2% — baseline ran, but no comparable score was available; uplift unavailable | 82.5% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.1% → 96.3% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 95.9% (-2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 84.7% → 89.7% (+5.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 87.1% → 89.6% (+2.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,21 +80,21 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,141,507 | 648,162 | +493,345 | +76.11% | skill 6/6; base 6/6 | -| claude-code | kpersist-01 | 228,709 | 109,286 | +119,423 | +109.28% | skill 1/1; base 1/1 | -| claude-code | kpersist-02 | 228,638 | 107,440 | +121,198 | +112.81% | skill 1/1; base 1/1 | -| claude-code | kpersist-03 | 232,288 | 108,070 | +124,218 | +114.94% | skill 1/1; base 1/1 | -| claude-code | kpersist-04 | 236,289 | 108,684 | +127,605 | +117.41% | skill 1/1; base 1/1 | -| claude-code | kpersist-05 | 71,294 | 107,376 | -36,082 | -33.60% | skill 1/1; base 1/1 | -| claude-code | kpersist-06 | 144,289 | 107,306 | +36,983 | +34.46% | skill 1/1; base 1/1 | -| codex | All cases | 360,535 | 580,343 | -219,808 | -37.88% | skill 6/6; base 6/6 | -| codex | kpersist-01 | 81,456 | 124,764 | -43,308 | -34.71% | skill 1/1; base 1/1 | -| codex | kpersist-02 | 42,527 | 13,385 | +29,142 | +217.72% | skill 1/1; base 1/1 | -| codex | kpersist-03 | 80,182 | 306,156 | -225,974 | -73.81% | skill 1/1; base 1/1 | -| codex | kpersist-04 | 68,889 | 35,219 | +33,670 | +95.60% | skill 1/1; base 1/1 | -| codex | kpersist-05 | 42,468 | 59,767 | -17,299 | -28.94% | skill 1/1; base 1/1 | -| codex | kpersist-06 | 45,013 | 41,052 | +3,961 | +9.65% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 1,502,042 | 1,228,505 | +273,537 | +22.27% | skill 12/12; base 12/12 | +| claude-code | All cases | 1,110,747 | 685,814 | +424,933 | +61.96% | skill 6/6; base 6/6 | +| claude-code | kpersist-01 | 316,113 | 147,635 | +168,478 | +114.12% | skill 1/1; base 1/1 | +| claude-code | kpersist-02 | 188,180 | 107,990 | +80,190 | +74.26% | skill 1/1; base 1/1 | +| claude-code | kpersist-03 | 193,261 | 108,036 | +85,225 | +78.89% | skill 1/1; base 1/1 | +| claude-code | kpersist-04 | 198,714 | 107,921 | +90,793 | +84.13% | skill 1/1; base 1/1 | +| claude-code | kpersist-05 | 107,232 | 106,780 | +452 | +0.42% | skill 1/1; base 1/1 | +| claude-code | kpersist-06 | 107,247 | 107,452 | -205 | -0.19% | skill 1/1; base 1/1 | +| codex | All cases | 362,860 | 395,309 | -32,449 | -8.21% | skill 6/6; base 6/6 | +| codex | kpersist-01 | 85,852 | 206,733 | -120,881 | -58.47% | skill 1/1; base 1/1 | +| codex | kpersist-02 | 42,500 | 13,551 | +28,949 | +213.63% | skill 1/1; base 1/1 | +| codex | kpersist-03 | 80,312 | 52,236 | +28,076 | +53.75% | skill 1/1; base 1/1 | +| codex | kpersist-04 | 70,257 | 34,827 | +35,430 | +101.73% | skill 1/1; base 1/1 | +| codex | kpersist-05 | 42,505 | 60,942 | -18,437 | -30.25% | skill 1/1; base 1/1 | +| codex | kpersist-06 | 41,434 | 27,020 | +14,414 | +53.35% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,473,607 | 1,081,123 | +392,484 | +36.30% | skill 12/12; base 12/12 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-persistence/skill-card.md b/skills/kumo-model-persistence/skill-card.md index c9fe4a13c..33fe6ce3f 100644 --- a/skills/kumo-model-persistence/skill-card.md +++ b/skills/kumo-model-persistence/skill-card.md @@ -75,12 +75,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.1% — baseline ran, but no comparable score was available; uplift unavailable | 80.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 91.7% → 100.0% (+8.3 points) | -| Correctness | 56.7% → 96.7% (+40.0 points) | 40.0% → 83.3% (+43.3 points) | -| Discoverability | 98.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 32.5% → 73.9% (+41.4 points) | 27.2% → 47.2% (+20.0 points) | -| Efficiency | 92.1% — baseline ran, but no comparable score was available; uplift unavailable | 82.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 91.8% — baseline ran, but no comparable score was available; uplift unavailable | 84.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 60.0% → 100.0% (+40.0 points) | 50.0% → 86.7% (+36.7 points) | +| Discoverability | 99.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 31.1% → 70.6% (+39.5 points) | 27.2% → 66.1% (+38.9 points) | +| Efficiency | 89.2% — baseline ran, but no comparable score was available; uplift unavailable | 82.5% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -95,9 +95,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.1% → 96.3% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 95.9% (-2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 84.7% → 89.7% (+5.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 87.1% → 89.6% (+2.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
0.1.0 (source: pyproject.toml)
diff --git a/skills/kumo-model-persistence/skill.oms.sig b/skills/kumo-model-persistence/skill.oms.sig index 82fd4c666..7165ff093 100644 --- a/skills/kumo-model-persistence/skill.oms.sig +++ b/skills/kumo-model-persistence/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1wZXJzaXN0ZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJiN2UxY2FlOTk0N2RhNTI5ZmIwM2Q5M2I1NDQ4NzYwNjRkZjQ4NTExY2RiMzQ0ODVmY2FmNDQ4YjllZTFjZmQzIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiNzg4MWU2MjlmYzUzNjNlMTI2MTFmNDA1Yzg4YzU5ZjlkODk0YTExZWFlOTk5ZWNlZTY0MDAxNWQ5NWExMmY3MyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJiMWU4MWU2ODg2ZDNkYTczZmMwMzI2ZjhmMGQ1YTYzYTBjNzIzNjlkNmYyNWZkM2M0MjkyN2U5YWUyMDBlZTI4IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiM2JiZjZmNDg1OGNmNGEwMWY4NTAzMThjNTliNThmNDUzMTkwNzQ3NTI1OTc4ZGI2ZjIxYTA2MjRlOTBiYmRlZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3JldGVudGlvbi5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIyYjA1NGFmMzNmODE2MmM1MThkZWE5ZjdhZDg0NGMwZDQxNjA3YjAwM2Q1ODIxZDUwYTJiZjY4YmU2Y2FjZjMyIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcmV1c2UubWQiLAogICAgICAgICJkaWdlc3QiOiAiMmM1YTUwMzFjODUwMTlmMTIxNWU2Yzc0MGRjZWRlN2E2ZTkyMDg1MmIzZjU3MmYwNGY5NGRhMGVjY2M4YWU4MSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3JldGFpbl9hcnRpZmFjdHMucHkiLAogICAgICAgICJkaWdlc3QiOiAiNGQzZDAzYjEwODlhMDI0MGQxZjkxM2MxZWQyZjhlNGFmOTg5ZWNiOGQzZTE5OWMzMmZkZGFlOTBmY2JhYWQzYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogImJjYzIxNjc2YjdiYmJkYTFlMjgzYmMwOTBmZTlhYzNmOTg5YzE3NDcyYjZkODM0OGRlMWRlM2I0YjQ5NzI0M2UiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMFeVa3Z/CPc/uXbxtNT/GKgFdtDCVUVpzN7l6x+BR21M5FMx4uIFY8WPzLTZvViK4gIwDG+C1VviG0kNZqvGtkS2E87P4RpF5lGhcCuS8Eu/VUU9Wg48YQErkqr3rkRYsE/w","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC1wZXJzaXN0ZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJjMWI1NTZkNmE3Y2VhMDdjNmI5YjkwNDFkMmE3OTJkMDI1OTI1NjM3YzczNTRkZDU5MjVjZTcwNWE2MDRmNWZjIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0sCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJkOTVjNzgwOGQwNjA0ZTNhNGExM2RkY2ZmYzFlMzgzNzJiYWM1MTEwMmIwNzlmNGQ5YjY5YWZlYjI1NWZmYjU4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogImIxZTgxZTY4ODZkM2RhNzNmYzAzMjZmOGYwZDVhNjNhMGM3MjM2OWQ2ZjI1ZmQzYzQyOTI3ZTlhZTIwMGVlMjgiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICIzYmJmNmY0ODU4Y2Y0YTAxZjg1MDMxOGM1OWI1OGY0NTMxOTA3NDc1MjU5NzhkYjZmMjFhMDYyNGU5MGJiZGVkIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcmV0ZW50aW9uLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjJiMDU0YWYzM2Y4MTYyYzUxOGRlYTlmN2FkODQ0YzBkNDE2MDdiMDAzZDU4MjFkNTBhMmJmNjhiZTZjYWNmMzIiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9yZXVzZS5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIyYzVhNTAzMWM4NTAxOWYxMjE1ZTZjNzQwZGNlZGU3YTZlOTIwODUyYjNmNTcyZjA0Zjk0ZGEwZWNjYzhhZTgxIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmV0YWluX2FydGlmYWN0cy5weSIsCiAgICAgICAgImRpZ2VzdCI6ICI0ZDNkMDNiMTA4OWEwMjQwZDFmOTEzYzFlZDJmOGU0YWY5ODllY2I4ZDNlMTk5YzMyZmRkYWU5MGZjYmFhZDNiIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJkaWdlc3QiOiAiOTE4NWVlOTIyOGZkZDU2MGIzNzlmMWZiMTk4MzkzMDhhNmJhYjk5YWI5NTE0YTNlNTc2OWFiYzM2NGQwMWY5MCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDFwNXWmR9zSC2BMMe6YFT3BTNgfYTn6KCQdk6YwFNsAS/T+eP74sTekYOVCh1RCysCMQCNYJ9rXvKDY/3rWnytxybXf+OBFGkWkA+9SVVj/tiMeqMUSUGGH/mKRGCaBMoDc4U=","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-model-tuning/BENCHMARK.md b/skills/kumo-model-tuning/BENCHMARK.md index a1172e504..8437c70fc 100644 --- a/skills/kumo-model-tuning/BENCHMARK.md +++ b/skills/kumo-model-tuning/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 99.8% — baseline ran, but no comparable score was available; uplift unavailable | 94.3% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | -| Correctness | 72.0% → 100.0% (+28.0 points) | 60.0% → 100.0% (+40.0 points) | +| Overall | 99.7% — baseline ran, but no comparable score was available; uplift unavailable | 93.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 60.0% → 96.0% (+36.0 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 54.2% → 100.0% (+45.8 points) | 51.7% → 85.8% (+34.1 points) | -| Efficiency | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 54.2% → 100.0% (+45.8 points) | 51.7% → 92.5% (+40.8 points) | +| Efficiency | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 87.5% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.8% → 97.6% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.9% → 97.4% (-0.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.9% → 92.3% (+1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 91.2% → 91.6% (+0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 655,712 | 869,059 | -213,347 | -24.55% | skill 5/5; base 5/5 | -| claude-code | ktune-01 | 109,028 | 71,668 | +37,360 | +52.13% | skill 1/1; base 1/1 | -| claude-code | ktune-02 | 148,001 | 179,352 | -31,351 | -17.48% | skill 1/1; base 1/1 | -| claude-code | ktune-03 | 109,759 | 178,796 | -69,037 | -38.61% | skill 1/1; base 1/1 | -| claude-code | ktune-04 | 108,902 | 107,501 | +1,401 | +1.30% | skill 1/1; base 1/1 | -| claude-code | ktune-05 | 180,022 | 331,742 | -151,720 | -45.73% | skill 1/1; base 1/1 | -| codex | All cases | 262,464 | 260,559 | +1,905 | +0.73% | skill 5/5; base 5/5 | -| codex | ktune-01 | 41,815 | 57,196 | -15,381 | -26.89% | skill 1/1; base 1/1 | -| codex | ktune-02 | 72,462 | 55,253 | +17,209 | +31.15% | skill 1/1; base 1/1 | -| codex | ktune-03 | 57,242 | 54,477 | +2,765 | +5.08% | skill 1/1; base 1/1 | -| codex | ktune-04 | 45,814 | 48,744 | -2,930 | -6.01% | skill 1/1; base 1/1 | -| codex | ktune-05 | 45,131 | 44,889 | +242 | +0.54% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 918,176 | 1,129,618 | -211,442 | -18.72% | skill 10/10; base 10/10 | +| claude-code | All cases | 732,189 | 869,250 | -137,061 | -15.77% | skill 5/5; base 5/5 | +| claude-code | ktune-01 | 108,983 | 71,489 | +37,494 | +52.45% | skill 1/1; base 1/1 | +| claude-code | ktune-02 | 148,053 | 179,038 | -30,985 | -17.31% | skill 1/1; base 1/1 | +| claude-code | ktune-03 | 148,030 | 179,219 | -31,189 | -17.40% | skill 1/1; base 1/1 | +| claude-code | ktune-04 | 109,082 | 107,470 | +1,612 | +1.50% | skill 1/1; base 1/1 | +| claude-code | ktune-05 | 218,041 | 332,034 | -113,993 | -34.33% | skill 1/1; base 1/1 | +| codex | All cases | 283,923 | 229,476 | +54,447 | +23.73% | skill 5/5; base 5/5 | +| codex | ktune-01 | 45,824 | 53,340 | -7,516 | -14.09% | skill 1/1; base 1/1 | +| codex | ktune-02 | 56,730 | 41,513 | +15,217 | +36.66% | skill 1/1; base 1/1 | +| codex | ktune-03 | 71,446 | 54,363 | +17,083 | +31.42% | skill 1/1; base 1/1 | +| codex | ktune-04 | 45,995 | 35,446 | +10,549 | +29.76% | skill 1/1; base 1/1 | +| codex | ktune-05 | 63,928 | 44,814 | +19,114 | +42.65% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,016,112 | 1,098,726 | -82,614 | -7.52% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-model-tuning/skill-card.md b/skills/kumo-model-tuning/skill-card.md index 4ee815a06..7f77fea41 100644 --- a/skills/kumo-model-tuning/skill-card.md +++ b/skills/kumo-model-tuning/skill-card.md @@ -65,33 +65,47 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
-**Overall verdict: PASS — Recommended for publication** +> ✅ **Overall verdict: PASS — Recommended for publication** -**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. ### Results at a Glance | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 99.8% — baseline ran, but no comparable score was available; uplift unavailable | 94.3% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | -| Correctness | 72.0% → 100.0% (+28.0 points) | 60.0% → 100.0% (+40.0 points) | +| Overall | 99.7% — baseline ran, but no comparable score was available; uplift unavailable | 93.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 60.0% → 96.0% (+36.0 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 54.2% → 100.0% (+45.8 points) | 51.7% → 85.8% (+34.1 points) | -| Efficiency | 98.8% — baseline ran, but no comparable score was available; uplift unavailable | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 54.2% → 100.0% (+45.8 points) | 51.7% → 92.5% (+40.8 points) | +| Efficiency | 98.7% — baseline ran, but no comparable score was available; uplift unavailable | 87.5% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. -Baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. A partial dimension was calculated from only the available configured signals. +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. -### Comparison details +#### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 97.8% → 97.6% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.9% → 97.4% (-0.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.9% → 92.3% (+1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 91.2% → 91.6% (+0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 11 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | ## Skill Version(s):
0.1.0 (source: pyproject.toml)
diff --git a/skills/kumo-model-tuning/skill.oms.sig b/skills/kumo-model-tuning/skill.oms.sig index 75f93cb42..ab347689e 100644 --- a/skills/kumo-model-tuning/skill.oms.sig +++ b/skills/kumo-model-tuning/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC10dW5pbmciLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiM2RkY2Y3MzlmODE2YzdlNjRlZGUyZmViNmI5NDU2ODFmZDFlMmJiZGZiMWI1Njc3NDg2YWUyODQ1ZGJmNDFhYiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0sCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3NGE4ZTdjNjU2ZTEzMzlkYTQwMDJkMjc5ZmRhNzE5NjE3ZmJiNjVlMWEzOGY3NGViYmNiNGMyMjlhMmU4ZDM0IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxNjQwMWRiY2Y5OTM0ZTY1MTkyNDdhNDNiN2VkYTYxNzc5NjRiMTJmNzBiNjg2MTY0MWU0NDE2MGU2ZDhlN2FiIiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjU4YjZjZjMyMjQyNTNlZWU1ZjljMWUwM2IxZjgyZTNhOTVlZThmMWM1MjdjMjVjYmIzNWMxNzU0ODkwZGRmZWIiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzZWI0MmIwZjg5NTNkMTcwMDI1NThkMDVhODgyODUxOTMxZDhjYjUxMGNhOGY3ZTVjNjBlYjljZjY3OTAyYzRkIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BsYW5zLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMDJjODk0MDcyYTk3MGQ1YjEwNTI0Njg5ZjU2YzhjMmIzYWNhNmQ2NGFiZGMyNTAwODE2ZTMwNDhiZTRmZjlkZiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMC7ieYP+ZUWMcaEG7uCDuy5ynS4sgInN5TgIYLyNiPp3RJXg7xf9oaUJYYUaLPTodQIxAMnXqaj3BOFKMCfcYFdBXUDViZUTlWLh2wcxd7q6t3gGA4KUu8yaVriZsnuTgQsJiw==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby1tb2RlbC10dW5pbmciLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiZmQ0MTVjZmY5NmE0OWU3YTljZDdkM2M4NjYyMTYyYWE4ODg4MTY4ODE3MjY3ZmRiZThkOTVjOGM1MTlmZTRhYiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIKICAgICAgXQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImVkMGRhOTA0ZDFlOTFkODhiOTJmMzU1MWU5ODhjNTA3MzA3M2E4OTM2M2JjZTNhNDEwNWM2OWZhNmUyZmJkMzYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjE2NDAxZGJjZjk5MzRlNjUxOTI0N2E0M2I3ZWRhNjE3Nzk2NGIxMmY3MGI2ODYxNjQxZTQ0MTYwZTZkOGU3YWIiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNThiNmNmMzIyNDI1M2VlZTVmOWMxZTAzYjFmODJlM2E5NWVlOGYxYzUyN2MyNWNiYjM1YzE3NTQ4OTBkZGZlYiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjNlYjQyYjBmODk1M2QxNzAwMjU1OGQwNWE4ODI4NTE5MzFkOGNiNTEwY2E4ZjdlNWM2MGViOWNmNjc5MDJjNGQiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcGxhbnMubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI3NGEzM2RmNjNhNWFmZWM1MGFkYTBkMGI5M2ZjZWVjNjY1MTNjOTlmNDA2ZGJhNTdlNDM2ZjY5NTgzOGJiN2M3IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMGbmafVTd30kLAVh/UzN1fSW4M/1RWYg/EQIXJTmmhWQjjPiHpB6Q0+Ek7otg6OnSwIwZsUuyThWMcHEU5wgE1qQTvKUMpNlN/wPwREkfJ6CKV2/3LFJjD3fH/dCzUkqO1JX","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumo-training-diagnostics/BENCHMARK.md b/skills/kumo-training-diagnostics/BENCHMARK.md index ee12120e1..a65880dee 100644 --- a/skills/kumo-training-diagnostics/BENCHMARK.md +++ b/skills/kumo-training-diagnostics/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 81.6% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 81.8% — baseline ran, but no comparable score was available; uplift unavailable | 87.5% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | -| Correctness | 40.0% → 80.0% (+40.0 points) | 36.0% → 88.0% (+52.0 points) | -| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 40.7% → 48.5% (+7.8 points) | 50.2% → 63.2% (+13.0 points) | -| Efficiency | 94.4% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 44.0% → 80.0% (+36.0 points) | 28.0% → 84.0% (+56.0 points) | +| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 38.2% → 48.5% (+10.3 points) | 48.2% → 65.2% (+17.0 points) | +| Efficiency | 95.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.4% → 97.2% (-1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 96.9% (-1.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.8% → 92.0% (+1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 92.9% → 91.9% (-1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 640,415 | 537,025 | +103,390 | +19.25% | skill 5/5; base 5/5 | -| claude-code | kdiag-01 | 197,077 | 107,167 | +89,910 | +83.90% | skill 1/1; base 1/1 | -| claude-code | kdiag-02 | 155,096 | 107,338 | +47,758 | +44.49% | skill 1/1; base 1/1 | -| claude-code | kdiag-03 | 72,991 | 106,956 | -33,965 | -31.76% | skill 1/1; base 1/1 | -| claude-code | kdiag-04 | 107,809 | 144,238 | -36,429 | -25.26% | skill 1/1; base 1/1 | -| claude-code | kdiag-05 | 107,442 | 71,326 | +36,116 | +50.64% | skill 1/1; base 1/1 | -| codex | All cases | 273,653 | 222,237 | +51,416 | +23.14% | skill 5/5; base 5/5 | -| codex | kdiag-01 | 61,253 | 27,099 | +34,154 | +126.03% | skill 1/1; base 1/1 | -| codex | kdiag-02 | 79,572 | 48,830 | +30,742 | +62.96% | skill 1/1; base 1/1 | -| codex | kdiag-03 | 27,911 | 55,282 | -27,371 | -49.51% | skill 1/1; base 1/1 | -| codex | kdiag-04 | 59,760 | 45,959 | +13,801 | +30.03% | skill 1/1; base 1/1 | -| codex | kdiag-05 | 45,157 | 45,067 | +90 | +0.20% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 914,068 | 759,262 | +154,806 | +20.39% | skill 10/10; base 10/10 | +| claude-code | All cases | 669,231 | 719,075 | -49,844 | -6.93% | skill 5/5; base 5/5 | +| claude-code | kdiag-01 | 115,117 | 142,507 | -27,390 | -19.22% | skill 1/1; base 1/1 | +| claude-code | kdiag-02 | 155,275 | 143,273 | +12,002 | +8.38% | skill 1/1; base 1/1 | +| claude-code | kdiag-03 | 145,921 | 144,279 | +1,642 | +1.14% | skill 1/1; base 1/1 | +| claude-code | kdiag-04 | 145,269 | 107,380 | +37,889 | +35.28% | skill 1/1; base 1/1 | +| claude-code | kdiag-05 | 107,649 | 181,636 | -73,987 | -40.73% | skill 1/1; base 1/1 | +| codex | All cases | 292,095 | 183,581 | +108,514 | +59.11% | skill 5/5; base 5/5 | +| codex | kdiag-01 | 61,235 | 41,122 | +20,113 | +48.91% | skill 1/1; base 1/1 | +| codex | kdiag-02 | 61,682 | 40,838 | +20,844 | +51.04% | skill 1/1; base 1/1 | +| codex | kdiag-03 | 61,576 | 42,577 | +18,999 | +44.62% | skill 1/1; base 1/1 | +| codex | kdiag-04 | 62,122 | 41,207 | +20,915 | +50.76% | skill 1/1; base 1/1 | +| codex | kdiag-05 | 45,480 | 17,837 | +27,643 | +154.98% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 961,326 | 902,656 | +58,670 | +6.50% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumo-training-diagnostics/skill-card.md b/skills/kumo-training-diagnostics/skill-card.md index de05202ed..ed9d08463 100644 --- a/skills/kumo-training-diagnostics/skill-card.md +++ b/skills/kumo-training-diagnostics/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Evaluate an existing Kumo training run in a Python notebook using aggregate metrics, trial results, learning curves, refit evidence, and training progress.
+Evaluate an existing Kumo training run in a Python notebook using aggregate metrics, trial results, learning curves, refit evidence, and training progress, inspecting existing results without automatically tuning, rerunning, or deploying a model.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data scientists use this skill to answer model-quality and training-diagnostic questions about an existing Kumo training run in a Snowflake notebook, inspecting recorded metrics, trial selection, learning curves, and progress without tuning, rerunning, or deploying a model.
+Developers and data scientists use this skill to answer model-quality and training-diagnostics questions about an existing Kumo training run in a notebook, such as available metrics, trial selection, overfitting from learning curves, and training progress, without retraining or tuning.
### Deployment Geography for Use:
Global
@@ -25,13 +25,14 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [Diagnostic workflow](references/diagnostics.md)
+- [Diagnostic workflow (references/diagnostics.md)](references/diagnostics.md)
+- [Kumo model persistence skill](../kumo-model-persistence/SKILL.md)
- [Kumo Engine documentation](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/overview)
## Skill Output:
**Output Type(s):** [Analysis, Code]
-**Output Format:** [Markdown with inline Python notebook code blocks]
+**Output Format:** [Markdown with inline Python code blocks]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [None]
@@ -42,15 +43,15 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each run in its own isolated sandbox pod (`k8s-sandbox`); each task was also attempted without the skill as a baseline.
+5 evaluation tasks (4 positive, 1 negative), 1 attempt per task, each in an isolated sandbox pod; every task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
- Security: Is it safe to use? Scored from `security` (100%).
- Correctness: Is the answer correct? Scored from `accuracy` (100%).
- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
-- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
-- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` (50%) and `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? `skill_efficiency` (50%) plus `token_efficiency` (50%).
Underlying evaluation signals used in this run:
- `security`: Unsafe operations, secret leakage, and unauthorized access.
@@ -74,12 +75,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 81.6% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 81.8% — baseline ran, but no comparable score was available; uplift unavailable | 87.5% — baseline ran, but no comparable score was available; uplift unavailable | | Security | 100.0% → 100.0% (±0.0 points) | 80.0% → 100.0% (+20.0 points) | -| Correctness | 40.0% → 80.0% (+40.0 points) | 36.0% → 88.0% (+52.0 points) | -| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 40.7% → 48.5% (+7.8 points) | 50.2% → 63.2% (+13.0 points) | -| Efficiency | 94.4% — baseline ran, but no comparable score was available; uplift unavailable | 96.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Correctness | 44.0% → 80.0% (+36.0 points) | 28.0% → 84.0% (+56.0 points) | +| Discoverability | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | 92.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 38.2% → 48.5% (+10.3 points) | 48.2% → 65.2% (+17.0 points) | +| Efficiency | 95.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -87,16 +88,24 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. -### Comparison details +#### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.4% → 97.2% (-1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 96.9% (-1.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.8% → 92.0% (+1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 92.9% → 91.9% (-1.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 12 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | ## Skill Version(s):
0.1.0 (source: pyproject.toml)
diff --git a/skills/kumo-training-diagnostics/skill.oms.sig b/skills/kumo-training-diagnostics/skill.oms.sig index bb7cdb6ee..cbff585c8 100644 --- a/skills/kumo-training-diagnostics/skill.oms.sig +++ b/skills/kumo-training-diagnostics/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby10cmFpbmluZy1kaWFnbm9zdGljcyIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIyODUzMTE0ODk5MmU1MTYyY2I5YmJhMjRkMmNhN2M1ZjVlNWY4MDNiZGQwYzdlNDFkNTg1MjcwZjkxN2UxNDVmIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI5Mjg0ZDI0N2UwMmQxYTljMGE4MzBkNjk3MmVmMjdhODM0NDliZjM5MjkzMjkxM2U4ODIzNzViZTI1OTIxY2JhIiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI2NGRkODhhZTY2MDJmYjFmMjdhNGY0M2E0ZTQyZDYwN2Y3M2ZkOWYxN2FhNDg4MTVhMzhjZTlmNDM0MGZlZjc2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjhjNGM3YjNhNjlkZDE0NzFhMDNiMjU3M2I3YmVlMTk0MTEyOWI3YTUyYzg5OThlYjg5NzcyMmRmMjE1NTI5NTQiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlNzVkMzBlNTQ1NTMyMmY4NWM1NDBkOGI0MzdkYjBjMDA3NjFkNTI3NThhNTJmM2I5YThhNzY4MDM2MWZjZTMwIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2RpYWdub3N0aWNzLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMWNlMGIxOGUyY2U2NjdiNTNkNDhiZTFiOWY5YmQwOWJmNDY3YWNiNTJjM2UxMjRiZWI5NjEzMDQ5MmU3OGQ2YiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMESrJBwoNsJnGplr9bI7NBkUoRZtjpYF1mVuXDhoNyYSHUg1PR7gLHTWRwTswxAwDwIwbKG4AoDyJRMUvN2rC9LAcFjiOCuodOpuOBaY0n+JB1iUGPfnxj9QnvSNFsalDkjs","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtby10cmFpbmluZy1kaWFnbm9zdGljcyIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJhYjA1OGUwNzFjNWM5YzQ2ODRhNDBkYTFiM2RiMDU3YmE0MjU2ZGMzYzQ5NDUxZGZhM2Q5ZTEyMDk1ODA4YTI1IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImU1OWNiZmZhY2M3ZmQzY2Q3OGIzYTM4M2RmNjk2YTA0ZTNkYjYxZGM4NTkzM2RhYzMzM2Y4MWFlY2I2YmYzMWIiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjY0ZGQ4OGFlNjYwMmZiMWYyN2E0ZjQzYTRlNDJkNjA3ZjczZmQ5ZjE3YWE0ODgxNWEzOGNlOWY0MzQwZmVmNzYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiOGM0YzdiM2E2OWRkMTQ3MWEwM2IyNTczYjdiZWUxOTQxMTI5YjdhNTJjODk5OGViODk3NzIyZGYyMTU1Mjk1NCIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImU3NWQzMGU1NDU1MzIyZjg1YzU0MGQ4YjQzN2RiMGMwMDc2MWQ1Mjc1OGE1MmYzYjlhOGE3NjgwMzYxZmNlMzAiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvZGlhZ25vc3RpY3MubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3YjE0NDQyOTY0OWYzMWQ2MjA2MjViMzlhN2RjZDNkZDZiYjc3ZDQxOTAxZjRjYjNlMjU4NzY5NjRhNDRmZGNiIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMH7xfP7wx59LcPDt9xvOff4b5Qk+/AwI2l8I5ySGyPMnNE6cxLZLpfypSie6Bli1egIwNi2Z8kQiT7BrQqvSDwUy4pF3ybe/sWWQbp17dwzlfz4lqkDdAJJzPWOVwujxAaDM","keyid":""}]}} \ No newline at end of file diff --git a/skills/kumoai-reference/BENCHMARK.md b/skills/kumoai-reference/BENCHMARK.md index ece5aee44..27ab5fbdc 100644 --- a/skills/kumoai-reference/BENCHMARK.md +++ b/skills/kumoai-reference/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | 93.7% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 90.0% → 100.0% (+10.0 points) | -| Correctness | 68.0% → 100.0% (+32.0 points) | 84.0% → 100.0% (+16.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 50.2% → 82.9% (+32.7 points) | 60.0% → 89.3% (+29.3 points) | -| Efficiency | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | 84.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 90.1% — baseline ran, but no comparable score was available; uplift unavailable | 86.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 80.0% (-20.0 points) | 100.0% → 80.0% (-20.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 80.0% → 100.0% (+20.0 points) | +| Discoverability | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 52.2% → 84.0% (+31.8 points) | 60.2% → 71.6% (+11.4 points) | +| Efficiency | 90.1% — baseline ran, but no comparable score was available; uplift unavailable | 85.2% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 95.4% → 88.4% (-7.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.2% → 88.9% (-8.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 78.9% → 86.7% (+7.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 80.3% → 87.0% (+6.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,19 +80,19 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 1,436,485 | 1,128,611 | +307,874 | +27.28% | skill 5/5; base 5/5 | -| claude-code | kref-01 | 121,632 | 35,554 | +86,078 | +242.10% | skill 1/1; base 1/1 | -| claude-code | kref-02 | 534,256 | 188,249 | +346,007 | +183.80% | skill 1/1; base 1/1 | -| claude-code | kref-03 | 265,314 | 314,517 | -49,203 | -15.64% | skill 1/1; base 1/1 | -| claude-code | kref-04 | 365,817 | 110,088 | +255,729 | +232.30% | skill 1/1; base 1/1 | -| claude-code | kref-05 | 149,466 | 480,203 | -330,737 | -68.87% | skill 1/1; base 1/1 | -| codex | All cases | 756,110 | 575,543 | +180,567 | +31.37% | skill 5/5; base 5/5 | -| codex | kref-01 | 69,061 | 38,017 | +31,044 | +81.66% | skill 1/1; base 1/1 | -| codex | kref-02 | 219,983 | 134,324 | +85,659 | +63.77% | skill 1/1; base 1/1 | -| codex | kref-03 | 97,023 | 127,096 | -30,073 | -23.66% | skill 1/1; base 1/1 | -| codex | kref-04 | 119,192 | 48,860 | +70,332 | +143.95% | skill 1/1; base 1/1 | -| codex | kref-05 | 250,851 | 227,246 | +23,605 | +10.39% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 2,192,595 | 1,704,154 | +488,441 | +28.66% | skill 10/10; base 10/10 | +| claude-code | All cases | 1,422,067 | 516,770 | +905,297 | +175.18% | skill 5/5; base 5/5 | +| claude-code | kref-01 | 121,639 | 35,714 | +85,925 | +240.59% | skill 1/1; base 1/1 | +| claude-code | kref-02 | 511,856 | 149,172 | +362,684 | +243.13% | skill 1/1; base 1/1 | +| claude-code | kref-03 | 210,623 | 146,408 | +64,215 | +43.86% | skill 1/1; base 1/1 | +| claude-code | kref-04 | 312,657 | 38,324 | +274,333 | +715.83% | skill 1/1; base 1/1 | +| claude-code | kref-05 | 265,292 | 147,152 | +118,140 | +80.28% | skill 1/1; base 1/1 | +| codex | All cases | 683,131 | 650,550 | +32,581 | +5.01% | skill 5/5; base 5/5 | +| codex | kref-01 | 50,294 | 44,401 | +5,893 | +13.27% | skill 1/1; base 1/1 | +| codex | kref-02 | 166,708 | 347,313 | -180,605 | -52.00% | skill 1/1; base 1/1 | +| codex | kref-03 | 123,388 | 92,347 | +31,041 | +33.61% | skill 1/1; base 1/1 | +| codex | kref-04 | 119,304 | 47,349 | +71,955 | +151.97% | skill 1/1; base 1/1 | +| codex | kref-05 | 223,437 | 119,140 | +104,297 | +87.54% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,105,198 | 1,167,320 | +937,878 | +80.34% | skill 10/10; base 10/10 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. diff --git a/skills/kumoai-reference/skill-card.md b/skills/kumoai-reference/skill-card.md index efc46b7ac..0c5098882 100644 --- a/skills/kumoai-reference/skill-card.md +++ b/skills/kumoai-reference/skill-card.md @@ -79,12 +79,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | 93.7% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 100.0% (±0.0 points) | 90.0% → 100.0% (+10.0 points) | -| Correctness | 68.0% → 100.0% (+32.0 points) | 84.0% → 100.0% (+16.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 50.2% → 82.9% (+32.7 points) | 60.0% → 89.3% (+29.3 points) | -| Efficiency | 88.3% — baseline ran, but no comparable score was available; uplift unavailable | 84.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 90.1% — baseline ran, but no comparable score was available; uplift unavailable | 86.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 80.0% (-20.0 points) | 100.0% → 80.0% (-20.0 points) | +| Correctness | 68.0% → 100.0% (+32.0 points) | 80.0% → 100.0% (+20.0 points) | +| Discoverability | 96.3% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 52.2% → 84.0% (+31.8 points) | 60.2% → 71.6% (+11.4 points) | +| Efficiency | 90.1% — baseline ran, but no comparable score was available; uplift unavailable | 85.2% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -99,9 +99,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 95.4% → 88.4% (-7.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 97.2% → 88.9% (-8.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 78.9% → 86.7% (+7.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 80.3% → 87.0% (+6.7 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status @@ -112,7 +112,7 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | ## Skill Version(s):
-0.1.0 (source: pyproject.toml); git SHA bfbf737f, committed 2026-11-18
+0.1.0 (source: pyproject.toml); git SHA 2bbedbe6, committed 2026-11-18
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/kumoai-reference/skill.oms.sig b/skills/kumoai-reference/skill.oms.sig index 73ce3b80c..61c2c9666 100644 --- a/skills/kumoai-reference/skill.oms.sig +++ b/skills/kumoai-reference/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtb2FpLXJlZmVyZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI2Nzc5N2VlY2JmMThhN2NlOGViYmRhYTI0OGMyNDNjM2MyYTQ0ZmM5MGY2ZGFmODE1NDg1OGY3YjY2Y2RkMjUzIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIzNTg0ZjBiNjYyNzE1MDhkYjMwNDYwOGU0MzkxZjkxZjU3ZGE1MDBiYzcwYWM3MzVmZjM2Y2JhZTczODc1YWU3IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIwNDMxNjJiYWJjNzU1YWJhMTM1ODI1YWYzY2EzNzVkOGYwZDUxZDkwNGNkY2FiZTYzZjczOWE2M2Y4NGNkNGFmIiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImU4MGNhY2FjY2M2MDdiMjAwMWZjMTkwNDRkY2Q3MTA5MmI1YTA1NDNiZjU3NjMwMmRmZGYwODEyZjQ3OGRlNmMiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI4ZTg4ZTIzYjE3YTIwZTE3YTkyNzg2ODc0MmZjNDA1MDY0YTg4ZDE0NWYyY2I4ODM2YTg1MmUzNDAwNWU0YmJjIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2dyYXBoX2NvbnN0cnVjdGlvbi5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImUwNDFhZjVlOTJmYjY2ZjIyMTU1ZTQ1OTQyOTk2YWVmMzhlYWM0YTRjYTdkMTRiOWQ5Y2FlZThlYWY2YjQ5YjgiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMva3Vtb19maW5ldHVuaW5nX3Nkay5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImYyOWYzZDJkMmJjYTk1NTBkZTUyNWJiNWJjNjJlNTU4NzhlMGY1NGJhN2Y1ZjEwOWIwMmZjOTc4YmJiODVlYmEiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvb3V0cHV0cy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjQ1Nzg3ZGFjYjhlOTc3MmFlOTViMDU3NGYyOTc4MzQ4N2U0Y2NjOWI3MGFlZTFjOTA0ZDgwZWQyZTg1YzEzYWYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcHFsX3dyaXRpbmcubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJjMDljMjNiYTEwYzdhYTAwODk5NjQ2MDAwYTE2ZWRiMzJjODlkMjljNjM1OWU2MTRkYzlhYzZhMmVhOTIwZjA4IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3Rhc2tfam91cm5leXMubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI3OWI1NTVkYmYyZDQ3YzI5OGVjZTY0ZjRmNWQ4ODRkZTc3ZDRmNzZhNjUzYmE5ZTIxODMzMGMzZjFjMTQ3ZmI1IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3Ryb3VibGVzaG9vdGluZ19zaWduYWxzLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNmNhZmZlOTZmYWVjZmYwMTZjMWJiOTI0YjE2MTUyMzUzNGJkYWJiZDI1OGMwODNmYTVhMjRmYjc1M2QwYWJmYiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMGJA7BJyARc0pRTO14QTIRps54brXXTDCWPpVBvQqg//jqQ4nG896eCliSFndyO2ewIxANDI/oL2o387XCIGYC6eEZkQ4BlMFg95N6Eh/aouL2h5nGrymgvH35O2WdGc6878wA==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAia3Vtb2FpLXJlZmVyZW5jZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJkNWE1OWExYTc5YWMyYTY5M2E3OWE5ZmY5NTNiYjg3N2YyZTM0ZjQ2NjA0N2MxNWViY2NkNmUzY2Q4NmVlYjg3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNTk5M2M5MTIxZGY5ZDZkYzhmOTI2NDg4YWIzNDIzMTZhYTIzY2VhMjBmMjRmMDFmNjQ0YjQxYzQ5NDAyYjc5NCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMDQzMTYyYmFiYzc1NWFiYTEzNTgyNWFmM2NhMzc1ZDhmMGQ1MWQ5MDRjZGNhYmU2M2Y3MzlhNjNmODRjZDRhZiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlODBjYWNhY2NjNjA3YjIwMDFmYzE5MDQ0ZGNkNzEwOTJiNWEwNTQzYmY1NzYzMDJkZmRmMDgxMmY0NzhkZTZjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOGU4OGUyM2IxN2EyMGUxN2E5Mjc4Njg3NDJmYzQwNTA2NGE4OGQxNDVmMmNiODgzNmE4NTJlMzQwMDVlNGJiYyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvZ3JhcGhfY29uc3RydWN0aW9uLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlMDQxYWY1ZTkyZmI2NmYyMjE1NWU0NTk0Mjk5NmFlZjM4ZWFjNGE0Y2E3ZDE0YjlkOWNhZWU4ZWFmNmI0OWI4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9rdW1vX2ZpbmV0dW5pbmdfc2RrLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJmMjlmM2QyZDJiY2E5NTUwZGU1MjViYjViYzYyZTU1ODc4ZTBmNTRiYTdmNWYxMDliMDJmYzk3OGJiYjg1ZWJhIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9vdXRwdXRzLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI0NTc4N2RhY2I4ZTk3NzJhZTk1YjA1NzRmMjk3ODM0ODdlNGNjYzliNzBhZWUxYzkwNGQ4MGVkMmU4NWMxM2FmIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9wcWxfd3JpdGluZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzA5YzIzYmExMGM3YWEwMDg5OTY0NjAwMGExNmVkYjMyYzg5ZDI5YzYzNTllNjE0ZGM5YWM2YTJlYTkyMGYwOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdGFza19qb3VybmV5cy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNzliNTU1ZGJmMmQ0N2MyOThlY2U2NGY0ZjVkODg0ZGU3N2Q0Zjc2YTY1M2JhOWUyMTgzMzBjM2YxYzE0N2ZiNSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdHJvdWJsZXNob290aW5nX3NpZ25hbHMubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjEwNzExNTI0ODYxNjNhNTczYTY0NjUzNjI0ZGE4ZDg5NTM5MDg0YWE2NTFiZTg2MDQwOWU0NjJiM2I4NzM1NDQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQC2VkMzp+y8hSdcED0i4y3NkEcU1OeDsmAK3JGZr3tNC+/ovfO2F/8Q8Bnciq24vOoCMCgBP14M2N8Igj9QVXUi+tQuQ94dZBYZvedIdwQJ6C+R82fPhYoAxM8gbKr9dC2R7Q==","keyid":""}]}} \ No newline at end of file