From 7af499a7797699167cb14ffcaa73fe7e42517908 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Tue, 3 Nov 2026 18:23:53 -0500 Subject: [PATCH 01/19] feat(nvidia-skill-finder): use the Hermes built-in catalog under Hermes The finder told every agent to run `npx skills add nvidia/skills --list` for catalog lookup and `npx skills add ... --global --yes` for install. On Hermes that bypasses the built-in catalog and the skill security scan, and `--global` writes the default profile's skills directory regardless of the active profile. Branch both the catalog lookup and the install on the agent the skill is running in. Under Hermes use `hermes skills search --source nvidia` and `hermes skills install NVIDIA/skills/`; every other agent keeps the existing `npx` path. Identifier casing is significant -- `NVIDIA/...` resolves to the trusted first-party source while `nvidia/...` resolves to a community-trust mirror -- so the skill is told to copy the identifier from search output verbatim. Hermes search matches every word in the query, so note that one or two keywords work where a natural-language phrase returns nothing. Also update the plugin README, which still said Hermes support was coming soon, and release the plugin as 1.5.0. Signed-off-by: Jason Dudash --- plugins.d/nvidia-skills.yml | 2 +- .../nvidia-skills/.claude-plugin/plugin.json | 2 +- .../nvidia-skills/.codex-plugin/plugin.json | 2 +- .../nvidia-skills/.cursor-plugin/plugin.json | 2 +- plugins/nvidia-skills/README.md | 11 +++- plugins/nvidia-skills/plugin.json | 2 +- .../skills/nvidia-skill-finder/SKILL.md | 51 ++++++++++++++++--- .../nvidia-skill-finder/evals/evals.json | 8 +-- skills/nvidia-skill-finder/SKILL.md | 51 ++++++++++++++++--- skills/nvidia-skill-finder/evals/evals.json | 8 +-- 10 files changed, 113 insertions(+), 26 deletions(-) diff --git a/plugins.d/nvidia-skills.yml b/plugins.d/nvidia-skills.yml index d33b939e0..58fb55498 100644 --- a/plugins.d/nvidia-skills.yml +++ b/plugins.d/nvidia-skills.yml @@ -1,6 +1,6 @@ name: nvidia-skills -version: "1.4.1" +version: "1.5.0" description: Find the right NVIDIA skill for GPU acceleration, CUDA, AI agents, data loading, training, inference, robotics, Physical AI, Omniverse, simulation, and other NVIDIA-powered workflows. display_name: NVIDIA Skills diff --git a/plugins/nvidia-skills/.claude-plugin/plugin.json b/plugins/nvidia-skills/.claude-plugin/plugin.json index a90ea7076..3fd8a3947 100644 --- a/plugins/nvidia-skills/.claude-plugin/plugin.json +++ b/plugins/nvidia-skills/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "nvidia-skills", - "version": "1.4.1", + "version": "1.5.0", "description": "Find the right NVIDIA skill for GPU acceleration, CUDA, AI agents, data loading, training, inference, robotics, Physical AI, Omniverse, simulation, and other NVIDIA-powered workflows.", "displayName": "NVIDIA Skills", "author": { diff --git a/plugins/nvidia-skills/.codex-plugin/plugin.json b/plugins/nvidia-skills/.codex-plugin/plugin.json index b1121adfa..37fc83004 100644 --- a/plugins/nvidia-skills/.codex-plugin/plugin.json +++ b/plugins/nvidia-skills/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "nvidia-skills", - "version": "1.4.1", + "version": "1.5.0", "description": "Find the right NVIDIA skill for GPU acceleration, CUDA, AI agents, data loading, training, inference, robotics, Physical AI, Omniverse, simulation, and other NVIDIA-powered workflows.", "author": { "name": "NVIDIA", diff --git a/plugins/nvidia-skills/.cursor-plugin/plugin.json b/plugins/nvidia-skills/.cursor-plugin/plugin.json index 266302a38..50afe2c44 100644 --- a/plugins/nvidia-skills/.cursor-plugin/plugin.json +++ b/plugins/nvidia-skills/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "nvidia-skills", - "version": "1.4.1", + "version": "1.5.0", "description": "Find the right NVIDIA skill for GPU acceleration, CUDA, AI agents, data loading, training, inference, robotics, Physical AI, Omniverse, simulation, and other NVIDIA-powered workflows.", "author": { "name": "NVIDIA" diff --git a/plugins/nvidia-skills/README.md b/plugins/nvidia-skills/README.md index da26fc4cb..2ed5f7387 100644 --- a/plugins/nvidia-skills/README.md +++ b/plugins/nvidia-skills/README.md @@ -4,12 +4,21 @@ This plugin ships one skill, `nvidia-skill-finder`. It checks NVIDIA's live skil ## Availability -This plugin is available as a first-party plugin in the Claude Code, Codex, and Cursor marketplaces. Hermes support is coming soon. +This plugin is available as a first-party plugin in the Claude Code, Codex, Cursor, and Hermes marketplaces. You can also get it by adding the self-hosted `nvidia-official` marketplace from the `NVIDIA/skills` repo. ## What the finder runs +What it runs depends on the agent it is running in. + +**In Hermes**, it uses the built-in skills catalog and never calls `npx`: + +- **Catalog lookup.** `hermes skills search --source nvidia` (or `hermes skills browse`), which reads the catalog Hermes already ships. +- **Installs only with approval.** `hermes skills install NVIDIA/skills/`, which runs Hermes's own security scan and installs into the active profile. Hermes prompts for confirmation itself; the finder does not pass `--yes`. + +**In every other agent** (Claude Code, Codex, Cursor): + - **Catalog lookup.** The finder runs `npx skills add nvidia/skills --list` to read the live catalog, which needs Node.js and downloads the `skills` CLI from npm. If the CLI is unavailable, it reads the catalog from `github.com/NVIDIA/skills`, `build.nvidia.com/skills`, or `raw.githubusercontent.com` instead. - **Installs only with approval.** It never installs without asking. When the user agrees, it runs `npx skills add nvidia/skills --skill --global --yes`, which installs that skill from the `main` branch of `NVIDIA/skills` into the agent's global skills directory. - **Wider search.** If nothing in the NVIDIA catalog matches, it may suggest `npx skills find `, which searches skills outside NVIDIA's catalog. diff --git a/plugins/nvidia-skills/plugin.json b/plugins/nvidia-skills/plugin.json index 070697bce..40365f865 100644 --- a/plugins/nvidia-skills/plugin.json +++ b/plugins/nvidia-skills/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "nvidia-skills", - "version": "1.4.1", + "version": "1.5.0", "description": "Find the right NVIDIA skill for GPU acceleration, CUDA, AI agents, data loading, training, inference, robotics, Physical AI, Omniverse, simulation, and other NVIDIA-powered workflows.", "author": { "name": "NVIDIA", diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md index e9aa2f887..d255de423 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md @@ -100,6 +100,7 @@ catalog check, but it is not a substitute for checking the NVIDIA skills catalog Catalog check means one of: +- `hermes skills search --source nvidia` (Hermes only) - `npx skills add nvidia/skills --list` - `https://github.com/NVIDIA/skills/tree/main/skills` - `https://build.nvidia.com/skills` @@ -107,13 +108,36 @@ Catalog check means one of: 1. Check whether the relevant NVIDIA skill is already installed or already in context. If it is, hand off to that skill instead of recommending install. 2. Query the live catalog before naming a specific install target or giving the - main product answer. When shell access is available, attempt this command first: + main product answer. Pick the lookup that matches the agent you are running + in. + +**Running in Hermes Agent:** use the built-in catalog, which is already scanned +and profile-aware. Do not use `npx` here. + +```bash +hermes skills search --source nvidia +``` + +Add `--json` for full, untruncated identifiers, and `--limit` to cap results. +Search matches all the words in the query, so use one or two keywords +(`cuda`, `jetson`, `vehicle routing`) rather than a sentence — a long +natural-language phrase such as `cuda kernel optimization` returns nothing. +Run `hermes skills browse` to page through everything. If `--source nvidia` +returns no rows for several distinct keywords, fall back to the sources below. + +**Every other agent** (Claude Code, Codex, Cursor, or an unknown harness): use +the `skills` CLI. ```bash npx skills add nvidia/skills --list ``` -Use the fallback catalog sources only if the CLI is unavailable, blocked, or fails: +Do not run `hermes` commands outside Hermes. The `hermes` binary may exist on a +machine where you are running as a different agent; installing through it would +write to Hermes's skills directory, where the agent you are running in cannot +load the result. + +Use the fallback catalog sources only if the lookup above is unavailable, blocked, or fails: - https://github.com/NVIDIA/skills/tree/main/skills - https://build.nvidia.com/skills - https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json @@ -136,7 +160,21 @@ For a strong match that is not installed: The NVIDIA skill could help with this. Would you like me to install ? ``` -Use the active agent target when known: +**Running in Hermes Agent:** install through Hermes so the skill is security +scanned and lands in the active profile. + +```bash +hermes skills install NVIDIA/skills/ +``` + +Prefer the `Identifier` from the `hermes skills search` result, copied verbatim, +over assembling one by hand; it may be a longer path such as +`NVIDIA/skills/skills/`. Both resolve, but casing is significant: +`NVIDIA/...` resolves to the trusted first-party source, while `nvidia/...` +resolves to a community-trust mirror. Hermes prompts for confirmation itself, +so do not add `--yes` on the user's behalf. + +**Every other agent** — use the active agent target when known: ```bash npx skills add nvidia/skills --skill --agent codex --global --yes @@ -163,12 +201,13 @@ does not pick up newly installed skills immediately. Do not fabricate catalog entries or guess at skill names. If catalog lookup fails, say the catalog could not be checked and do not name a concrete slug or -emit an `npx skills add ... --skill ` install command. Offer to share the -catalog URL, retry lookup, or continue with general help. +emit an install command for it, in either form. Offer to share the catalog URL, +retry lookup, or continue with general help. ## When No Skills are Found Say that no strong NVIDIA catalog match was found, then either: -1. search the broader ecosystem with npx skills find , or +1. search the broader ecosystem — `hermes skills search ` without + `--source` in Hermes, otherwise `npx skills find ` — or 2. offer to help directly without a skill, or 3. suggest creating a new skill if the workflow is recurring. diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/evals/evals.json b/plugins/nvidia-skills/skills/nvidia-skill-finder/evals/evals.json index 7c351fc94..21fa07091 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/evals/evals.json +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/evals/evals.json @@ -7,7 +7,7 @@ "ground_truth": "The agent identifies Decision Optimization as a strong NVIDIA taxonomy match, checks the live NVIDIA skills catalog, and recommends a current cuOpt routing skill rather than implementing a routing solver from memory.", "expected_behavior": [ "Treats vehicle routing as a strong NVIDIA Decision Optimization signal", - "Checks the live NVIDIA catalog (e.g. `npx skills add nvidia/skills --list`) before naming a specific skill", + "Checks the live NVIDIA catalog before naming a specific skill, using the lookup for the current agent (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`)", "Recommends a cuOpt routing-related skill if present in the catalog", "Asks before installing the skill", "Does not implement a routing solver before offering the skill" @@ -99,7 +99,7 @@ "ground_truth": "The agent treats Jetson driver installation as a strong NVIDIA Robotics / Jetson workflow signal, checks the live NVIDIA catalog before naming a skill, and recommends the current Jetson setup or quick-start skill if available.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Jetson driver / JetPack / BSP request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends a Jetson setup skill (e.g. jetson-quick-start) if present", "Explains that Jetson drivers are normally bundled with JetPack / Jetson Linux rather than installed as standalone desktop drivers", @@ -114,7 +114,7 @@ "ground_truth": "The agent treats Jetson setup as a strong NVIDIA Robotics / Jetson workflow signal, checks the live NVIDIA skills catalog before using product docs as the main answer, and recommends the current Jetson setup or quick-start skill if available.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Jetson setup request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends a Jetson setup skill (e.g. jetson-quick-start) if present", "Provides practical Jetson setup guidance only after the catalog lookup path", @@ -129,7 +129,7 @@ "ground_truth": "The agent treats Isaac Lab as a strong NVIDIA Robotics / Physical AI signal, checks the live NVIDIA skills catalog before giving general Isaac Lab guidance, and recommends a current Isaac-related skill only if the catalog has one.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Isaac Lab request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends an Isaac-related skill if present in the catalog, or says no strong catalog match was found without inventing a skill name", "Asks before installing the skill" diff --git a/skills/nvidia-skill-finder/SKILL.md b/skills/nvidia-skill-finder/SKILL.md index e9aa2f887..d255de423 100644 --- a/skills/nvidia-skill-finder/SKILL.md +++ b/skills/nvidia-skill-finder/SKILL.md @@ -100,6 +100,7 @@ catalog check, but it is not a substitute for checking the NVIDIA skills catalog Catalog check means one of: +- `hermes skills search --source nvidia` (Hermes only) - `npx skills add nvidia/skills --list` - `https://github.com/NVIDIA/skills/tree/main/skills` - `https://build.nvidia.com/skills` @@ -107,13 +108,36 @@ Catalog check means one of: 1. Check whether the relevant NVIDIA skill is already installed or already in context. If it is, hand off to that skill instead of recommending install. 2. Query the live catalog before naming a specific install target or giving the - main product answer. When shell access is available, attempt this command first: + main product answer. Pick the lookup that matches the agent you are running + in. + +**Running in Hermes Agent:** use the built-in catalog, which is already scanned +and profile-aware. Do not use `npx` here. + +```bash +hermes skills search --source nvidia +``` + +Add `--json` for full, untruncated identifiers, and `--limit` to cap results. +Search matches all the words in the query, so use one or two keywords +(`cuda`, `jetson`, `vehicle routing`) rather than a sentence — a long +natural-language phrase such as `cuda kernel optimization` returns nothing. +Run `hermes skills browse` to page through everything. If `--source nvidia` +returns no rows for several distinct keywords, fall back to the sources below. + +**Every other agent** (Claude Code, Codex, Cursor, or an unknown harness): use +the `skills` CLI. ```bash npx skills add nvidia/skills --list ``` -Use the fallback catalog sources only if the CLI is unavailable, blocked, or fails: +Do not run `hermes` commands outside Hermes. The `hermes` binary may exist on a +machine where you are running as a different agent; installing through it would +write to Hermes's skills directory, where the agent you are running in cannot +load the result. + +Use the fallback catalog sources only if the lookup above is unavailable, blocked, or fails: - https://github.com/NVIDIA/skills/tree/main/skills - https://build.nvidia.com/skills - https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json @@ -136,7 +160,21 @@ For a strong match that is not installed: The NVIDIA skill could help with this. Would you like me to install ? ``` -Use the active agent target when known: +**Running in Hermes Agent:** install through Hermes so the skill is security +scanned and lands in the active profile. + +```bash +hermes skills install NVIDIA/skills/ +``` + +Prefer the `Identifier` from the `hermes skills search` result, copied verbatim, +over assembling one by hand; it may be a longer path such as +`NVIDIA/skills/skills/`. Both resolve, but casing is significant: +`NVIDIA/...` resolves to the trusted first-party source, while `nvidia/...` +resolves to a community-trust mirror. Hermes prompts for confirmation itself, +so do not add `--yes` on the user's behalf. + +**Every other agent** — use the active agent target when known: ```bash npx skills add nvidia/skills --skill --agent codex --global --yes @@ -163,12 +201,13 @@ does not pick up newly installed skills immediately. Do not fabricate catalog entries or guess at skill names. If catalog lookup fails, say the catalog could not be checked and do not name a concrete slug or -emit an `npx skills add ... --skill ` install command. Offer to share the -catalog URL, retry lookup, or continue with general help. +emit an install command for it, in either form. Offer to share the catalog URL, +retry lookup, or continue with general help. ## When No Skills are Found Say that no strong NVIDIA catalog match was found, then either: -1. search the broader ecosystem with npx skills find , or +1. search the broader ecosystem — `hermes skills search ` without + `--source` in Hermes, otherwise `npx skills find ` — or 2. offer to help directly without a skill, or 3. suggest creating a new skill if the workflow is recurring. diff --git a/skills/nvidia-skill-finder/evals/evals.json b/skills/nvidia-skill-finder/evals/evals.json index 7c351fc94..21fa07091 100644 --- a/skills/nvidia-skill-finder/evals/evals.json +++ b/skills/nvidia-skill-finder/evals/evals.json @@ -7,7 +7,7 @@ "ground_truth": "The agent identifies Decision Optimization as a strong NVIDIA taxonomy match, checks the live NVIDIA skills catalog, and recommends a current cuOpt routing skill rather than implementing a routing solver from memory.", "expected_behavior": [ "Treats vehicle routing as a strong NVIDIA Decision Optimization signal", - "Checks the live NVIDIA catalog (e.g. `npx skills add nvidia/skills --list`) before naming a specific skill", + "Checks the live NVIDIA catalog before naming a specific skill, using the lookup for the current agent (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`)", "Recommends a cuOpt routing-related skill if present in the catalog", "Asks before installing the skill", "Does not implement a routing solver before offering the skill" @@ -99,7 +99,7 @@ "ground_truth": "The agent treats Jetson driver installation as a strong NVIDIA Robotics / Jetson workflow signal, checks the live NVIDIA catalog before naming a skill, and recommends the current Jetson setup or quick-start skill if available.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Jetson driver / JetPack / BSP request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends a Jetson setup skill (e.g. jetson-quick-start) if present", "Explains that Jetson drivers are normally bundled with JetPack / Jetson Linux rather than installed as standalone desktop drivers", @@ -114,7 +114,7 @@ "ground_truth": "The agent treats Jetson setup as a strong NVIDIA Robotics / Jetson workflow signal, checks the live NVIDIA skills catalog before using product docs as the main answer, and recommends the current Jetson setup or quick-start skill if available.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Jetson setup request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends a Jetson setup skill (e.g. jetson-quick-start) if present", "Provides practical Jetson setup guidance only after the catalog lookup path", @@ -129,7 +129,7 @@ "ground_truth": "The agent treats Isaac Lab as a strong NVIDIA Robotics / Physical AI signal, checks the live NVIDIA skills catalog before giving general Isaac Lab guidance, and recommends a current Isaac-related skill only if the catalog has one.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Isaac Lab request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends an Isaac-related skill if present in the catalog, or says no strong catalog match was found without inventing a skill name", "Asks before installing the skill" From 212ec3c1db900cc00830d8385743f25782413021 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Tue, 3 Nov 2026 19:43:54 -0500 Subject: [PATCH 02/19] fix(nvidia-skill-finder): keep catalog recall under Hermes search `hermes skills search` requires every word in the query to match, so a descriptive phrase usually returns nothing: `jetson pinmux` returns no rows even though `jetson-customize-pinmux` exists, and `OpenUSD` does not return `omniverse-usd-performance-tuning`. Across ten realistic lookups only six found the right skill. That is a real regression against the `npx` path, where the model sees all 431 names and descriptions and does the matching itself. Searching one broad product, family, or lane keyword at a time recovers every one of those misses, so require single-keyword queries and tell the model to run two or three of them and choose from the full descriptions `--json` returns rather than trusting the first row. Add a full-catalog fallback for when those searches still come up empty, so a narrow search term cannot hide a skill that exists. `hermes skills browse` is not used for this: it truncates both names and descriptions and costs more context than the catalog listing it would replace. Signed-off-by: Jason Dudash --- plugins/nvidia-skills/README.md | 2 +- .../skills/nvidia-skill-finder/SKILL.md | 30 ++++++++++++++----- skills/nvidia-skill-finder/SKILL.md | 30 ++++++++++++++----- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/plugins/nvidia-skills/README.md b/plugins/nvidia-skills/README.md index 2ed5f7387..e9973ad34 100644 --- a/plugins/nvidia-skills/README.md +++ b/plugins/nvidia-skills/README.md @@ -14,7 +14,7 @@ What it runs depends on the agent it is running in. **In Hermes**, it uses the built-in skills catalog and never calls `npx`: -- **Catalog lookup.** `hermes skills search --source nvidia` (or `hermes skills browse`), which reads the catalog Hermes already ships. +- **Catalog lookup.** `hermes skills search --source nvidia --json`, which reads the catalog Hermes already ships. That search needs every word in the query to match, so the finder queries one broad keyword at a time. If several keywords come back empty it reads the full catalog listing from `raw.githubusercontent.com` and picks from that, so a narrow search term can't hide a skill that exists. - **Installs only with approval.** `hermes skills install NVIDIA/skills/`, which runs Hermes's own security scan and installs into the active profile. Hermes prompts for confirmation itself; the finder does not pass `--yes`. **In every other agent** (Claude Code, Codex, Cursor): diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md index d255de423..b271c71cc 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md @@ -100,7 +100,7 @@ catalog check, but it is not a substitute for checking the NVIDIA skills catalog Catalog check means one of: -- `hermes skills search --source nvidia` (Hermes only) +- `hermes skills search --source nvidia --json` (Hermes only) - `npx skills add nvidia/skills --list` - `https://github.com/NVIDIA/skills/tree/main/skills` - `https://build.nvidia.com/skills` @@ -115,15 +115,29 @@ Catalog check means one of: and profile-aware. Do not use `npx` here. ```bash -hermes skills search --source nvidia +hermes skills search --source nvidia --json ``` -Add `--json` for full, untruncated identifiers, and `--limit` to cap results. -Search matches all the words in the query, so use one or two keywords -(`cuda`, `jetson`, `vehicle routing`) rather than a sentence — a long -natural-language phrase such as `cuda kernel optimization` returns nothing. -Run `hermes skills browse` to page through everything. If `--source nvidia` -returns no rows for several distinct keywords, fall back to the sources below. +Search requires every word in the query to match, so query one broad product, +family, or lane keyword at a time: `jetson`, `usd`, `dynamo`, `pandas`, `cuda`. +Descriptive phrases usually return nothing — `jetson pinmux` returns no rows +even though `jetson-customize-pinmux` exists, while `jetson` returns it. Run two +or three separate single-keyword searches (product name, product family, +taxonomy lane) instead of one phrase. + +`--json` returns each candidate's full description. Read them and choose the +match yourself; do not assume the first row is the answer. This search ranks +lexically, so it narrows the catalog but does not do the matching for you. + +If several single-keyword searches return nothing, list the whole catalog and +choose from it before giving up: + +```bash +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json | jq -r '.skills[] | "\(.name): \(.description)"' +``` + +Confirm the skill you picked with `hermes skills inspect `, then +install it through Hermes. **Every other agent** (Claude Code, Codex, Cursor, or an unknown harness): use the `skills` CLI. diff --git a/skills/nvidia-skill-finder/SKILL.md b/skills/nvidia-skill-finder/SKILL.md index d255de423..b271c71cc 100644 --- a/skills/nvidia-skill-finder/SKILL.md +++ b/skills/nvidia-skill-finder/SKILL.md @@ -100,7 +100,7 @@ catalog check, but it is not a substitute for checking the NVIDIA skills catalog Catalog check means one of: -- `hermes skills search --source nvidia` (Hermes only) +- `hermes skills search --source nvidia --json` (Hermes only) - `npx skills add nvidia/skills --list` - `https://github.com/NVIDIA/skills/tree/main/skills` - `https://build.nvidia.com/skills` @@ -115,15 +115,29 @@ Catalog check means one of: and profile-aware. Do not use `npx` here. ```bash -hermes skills search --source nvidia +hermes skills search --source nvidia --json ``` -Add `--json` for full, untruncated identifiers, and `--limit` to cap results. -Search matches all the words in the query, so use one or two keywords -(`cuda`, `jetson`, `vehicle routing`) rather than a sentence — a long -natural-language phrase such as `cuda kernel optimization` returns nothing. -Run `hermes skills browse` to page through everything. If `--source nvidia` -returns no rows for several distinct keywords, fall back to the sources below. +Search requires every word in the query to match, so query one broad product, +family, or lane keyword at a time: `jetson`, `usd`, `dynamo`, `pandas`, `cuda`. +Descriptive phrases usually return nothing — `jetson pinmux` returns no rows +even though `jetson-customize-pinmux` exists, while `jetson` returns it. Run two +or three separate single-keyword searches (product name, product family, +taxonomy lane) instead of one phrase. + +`--json` returns each candidate's full description. Read them and choose the +match yourself; do not assume the first row is the answer. This search ranks +lexically, so it narrows the catalog but does not do the matching for you. + +If several single-keyword searches return nothing, list the whole catalog and +choose from it before giving up: + +```bash +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json | jq -r '.skills[] | "\(.name): \(.description)"' +``` + +Confirm the skill you picked with `hermes skills inspect `, then +install it through Hermes. **Every other agent** (Claude Code, Codex, Cursor, or an unknown harness): use the `skills` CLI. From 8c346f807c97753684484a896872fb2cc452cfcb Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Wed, 4 Nov 2026 11:02:45 -0500 Subject: [PATCH 03/19] refactor(nvidia-skill-finder): move Hermes recall guidance to a reference The Hermes branch had grown to roughly 50 inline lines, most of it recovery detail for the substring-match behaviour of hermes skills search. Keep the correctness-critical parts inline -- the two commands, the NVIDIA/ vs nvidia/ casing rule, and the guard against running hermes outside Hermes -- and move the recall tactics, full-catalog fallback, and identifier confirmation to references/hermes-catalog.md, loaded only when searches come up empty. This follows the progressive-disclosure pattern the skill already uses for references/taxonomy-routing.md. Signed-off-by: Jason Dudash --- .../skills/nvidia-skill-finder/SKILL.md | 33 +++------ .../references/hermes-catalog.md | 69 +++++++++++++++++++ skills/nvidia-skill-finder/SKILL.md | 33 +++------ .../references/hermes-catalog.md | 69 +++++++++++++++++++ 4 files changed, 154 insertions(+), 50 deletions(-) create mode 100644 plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md create mode 100644 skills/nvidia-skill-finder/references/hermes-catalog.md diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md index b271c71cc..2445b09b9 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md @@ -118,26 +118,11 @@ and profile-aware. Do not use `npx` here. hermes skills search --source nvidia --json ``` -Search requires every word in the query to match, so query one broad product, -family, or lane keyword at a time: `jetson`, `usd`, `dynamo`, `pandas`, `cuda`. -Descriptive phrases usually return nothing — `jetson pinmux` returns no rows -even though `jetson-customize-pinmux` exists, while `jetson` returns it. Run two -or three separate single-keyword searches (product name, product family, -taxonomy lane) instead of one phrase. - -`--json` returns each candidate's full description. Read them and choose the -match yourself; do not assume the first row is the answer. This search ranks -lexically, so it narrows the catalog but does not do the matching for you. - -If several single-keyword searches return nothing, list the whole catalog and -choose from it before giving up: - -```bash -curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json | jq -r '.skills[] | "\(.name): \(.description)"' -``` - -Confirm the skill you picked with `hermes skills inspect `, then -install it through Hermes. +Search one broad keyword at a time — `jetson`, `usd`, `dynamo`, `cuda` — not a +descriptive phrase, which usually returns nothing. Read the returned +descriptions and choose the match yourself. Read +[references/hermes-catalog.md](references/hermes-catalog.md) when those searches +come up empty or before installing an identifier you are unsure about. **Every other agent** (Claude Code, Codex, Cursor, or an unknown harness): use the `skills` CLI. @@ -181,11 +166,9 @@ scanned and lands in the active profile. hermes skills install NVIDIA/skills/ ``` -Prefer the `Identifier` from the `hermes skills search` result, copied verbatim, -over assembling one by hand; it may be a longer path such as -`NVIDIA/skills/skills/`. Both resolve, but casing is significant: -`NVIDIA/...` resolves to the trusted first-party source, while `nvidia/...` -resolves to a community-trust mirror. Hermes prompts for confirmation itself, +Casing is significant: `NVIDIA/...` resolves to the trusted first-party source, +while `nvidia/...` resolves to a community-trust mirror. Prefer the `Identifier` +from the search result, copied verbatim. Hermes prompts for confirmation itself, so do not add `--yes` on the user's behalf. **Every other agent** — use the active agent target when known: diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md new file mode 100644 index 000000000..a23dbc3cd --- /dev/null +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -0,0 +1,69 @@ +# Hermes Catalog Recovery + +Read this when `hermes skills search --source nvidia --json` returns +nothing useful, or when you are unsure whether the identifier you are about to +install is the trusted one. + +## Why single keywords work and phrases do not + +Hermes matches a query as a plain case-insensitive substring against each +entry's name, description, tags, identifier, and provider joined together. It +does not tokenize, so a multi-word query only matches when those exact words +appear consecutively in that order. + +The practical effect is that descriptive phrases collapse to zero results while +the obvious keyword works: + +| Query | Result | +| --- | --- | +| `jetson pinmux` | no rows, even though `jetson-customize-pinmux` exists | +| `jetson` | returns `jetson-customize-pinmux` and the rest of the family | +| `OpenUSD` | returns only this finder | +| `usd` | returns the Omniverse and USD skills | + +## Search strategy + +Run two or three separate single-keyword searches rather than one phrase, and +widen by one step each time: + +1. The product name as users say it — `jetson`, `dynamo`, `cuopt`. +2. The product family — `omniverse`, `rapids`, `nemo`, `holoscan`. +3. The taxonomy lane — `usd`, `routing`, `inference`, `genomics`. + +`--json` returns each candidate's full description. Read the descriptions and +choose the match yourself; do not assume the first row is the answer. The search +ranks lexically, so it narrows the catalog but does not do the matching for you. + +## Full-catalog fallback + +If several single-keyword searches return nothing, list the whole catalog and +choose from it before giving up: + +```bash +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ + | jq -r '.skills[] | "\(.name): \(.description)"' +``` + +Do not use `hermes skills browse` for this. It costs more context than the full +catalog listing above and truncates both names and descriptions, so it is the +one option that is worse on both accuracy and cost. + +## Confirming the identifier before install + +Confirm the skill you picked before installing it: + +```bash +hermes skills inspect +``` + +Prefer the `Identifier` from the search result, copied verbatim, over assembling +one by hand. It may be a longer path such as `NVIDIA/skills/skills/`; +that form and the short `NVIDIA/skills/` both resolve to the same +trusted skill with the same scan hash. + +Casing is what determines trust, not the number of path segments. `NVIDIA/...` +resolves to the trusted first-party source; `nvidia/...` resolves to a +community-trust mirror with a placeholder description. Always use `NVIDIA`. + +Hermes prompts for confirmation itself, so do not pass `--yes` on the user's +behalf. diff --git a/skills/nvidia-skill-finder/SKILL.md b/skills/nvidia-skill-finder/SKILL.md index b271c71cc..2445b09b9 100644 --- a/skills/nvidia-skill-finder/SKILL.md +++ b/skills/nvidia-skill-finder/SKILL.md @@ -118,26 +118,11 @@ and profile-aware. Do not use `npx` here. hermes skills search --source nvidia --json ``` -Search requires every word in the query to match, so query one broad product, -family, or lane keyword at a time: `jetson`, `usd`, `dynamo`, `pandas`, `cuda`. -Descriptive phrases usually return nothing — `jetson pinmux` returns no rows -even though `jetson-customize-pinmux` exists, while `jetson` returns it. Run two -or three separate single-keyword searches (product name, product family, -taxonomy lane) instead of one phrase. - -`--json` returns each candidate's full description. Read them and choose the -match yourself; do not assume the first row is the answer. This search ranks -lexically, so it narrows the catalog but does not do the matching for you. - -If several single-keyword searches return nothing, list the whole catalog and -choose from it before giving up: - -```bash -curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json | jq -r '.skills[] | "\(.name): \(.description)"' -``` - -Confirm the skill you picked with `hermes skills inspect `, then -install it through Hermes. +Search one broad keyword at a time — `jetson`, `usd`, `dynamo`, `cuda` — not a +descriptive phrase, which usually returns nothing. Read the returned +descriptions and choose the match yourself. Read +[references/hermes-catalog.md](references/hermes-catalog.md) when those searches +come up empty or before installing an identifier you are unsure about. **Every other agent** (Claude Code, Codex, Cursor, or an unknown harness): use the `skills` CLI. @@ -181,11 +166,9 @@ scanned and lands in the active profile. hermes skills install NVIDIA/skills/ ``` -Prefer the `Identifier` from the `hermes skills search` result, copied verbatim, -over assembling one by hand; it may be a longer path such as -`NVIDIA/skills/skills/`. Both resolve, but casing is significant: -`NVIDIA/...` resolves to the trusted first-party source, while `nvidia/...` -resolves to a community-trust mirror. Hermes prompts for confirmation itself, +Casing is significant: `NVIDIA/...` resolves to the trusted first-party source, +while `nvidia/...` resolves to a community-trust mirror. Prefer the `Identifier` +from the search result, copied verbatim. Hermes prompts for confirmation itself, so do not add `--yes` on the user's behalf. **Every other agent** — use the active agent target when known: diff --git a/skills/nvidia-skill-finder/references/hermes-catalog.md b/skills/nvidia-skill-finder/references/hermes-catalog.md new file mode 100644 index 000000000..a23dbc3cd --- /dev/null +++ b/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -0,0 +1,69 @@ +# Hermes Catalog Recovery + +Read this when `hermes skills search --source nvidia --json` returns +nothing useful, or when you are unsure whether the identifier you are about to +install is the trusted one. + +## Why single keywords work and phrases do not + +Hermes matches a query as a plain case-insensitive substring against each +entry's name, description, tags, identifier, and provider joined together. It +does not tokenize, so a multi-word query only matches when those exact words +appear consecutively in that order. + +The practical effect is that descriptive phrases collapse to zero results while +the obvious keyword works: + +| Query | Result | +| --- | --- | +| `jetson pinmux` | no rows, even though `jetson-customize-pinmux` exists | +| `jetson` | returns `jetson-customize-pinmux` and the rest of the family | +| `OpenUSD` | returns only this finder | +| `usd` | returns the Omniverse and USD skills | + +## Search strategy + +Run two or three separate single-keyword searches rather than one phrase, and +widen by one step each time: + +1. The product name as users say it — `jetson`, `dynamo`, `cuopt`. +2. The product family — `omniverse`, `rapids`, `nemo`, `holoscan`. +3. The taxonomy lane — `usd`, `routing`, `inference`, `genomics`. + +`--json` returns each candidate's full description. Read the descriptions and +choose the match yourself; do not assume the first row is the answer. The search +ranks lexically, so it narrows the catalog but does not do the matching for you. + +## Full-catalog fallback + +If several single-keyword searches return nothing, list the whole catalog and +choose from it before giving up: + +```bash +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ + | jq -r '.skills[] | "\(.name): \(.description)"' +``` + +Do not use `hermes skills browse` for this. It costs more context than the full +catalog listing above and truncates both names and descriptions, so it is the +one option that is worse on both accuracy and cost. + +## Confirming the identifier before install + +Confirm the skill you picked before installing it: + +```bash +hermes skills inspect +``` + +Prefer the `Identifier` from the search result, copied verbatim, over assembling +one by hand. It may be a longer path such as `NVIDIA/skills/skills/`; +that form and the short `NVIDIA/skills/` both resolve to the same +trusted skill with the same scan hash. + +Casing is what determines trust, not the number of path segments. `NVIDIA/...` +resolves to the trusted first-party source; `nvidia/...` resolves to a +community-trust mirror with a placeholder description. Always use `NVIDIA`. + +Hermes prompts for confirmation itself, so do not pass `--yes` on the user's +behalf. From b7c41505ec65cf94421d4dca5d91ac1fc4d13862 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Wed, 4 Nov 2026 11:28:34 -0500 Subject: [PATCH 04/19] refactor(nvidia-skill-finder): lead with the default path, point to Hermes Put the npx commands back first and unbroken so the common path reads as the default, and reduce the Hermes branch to one bullet in each of the lookup and install sections pointing at references/hermes-catalog.md. The reference now carries the whole Hermes flow rather than only the recovery tactics. Reorders the plugin README the same way. Signed-off-by: Jason Dudash --- plugins/nvidia-skills/README.md | 16 ++--- .../skills/nvidia-skill-finder/SKILL.md | 52 ++++----------- .../references/hermes-catalog.md | 65 +++++++++++-------- skills/nvidia-skill-finder/SKILL.md | 52 ++++----------- .../references/hermes-catalog.md | 65 +++++++++++-------- 5 files changed, 111 insertions(+), 139 deletions(-) diff --git a/plugins/nvidia-skills/README.md b/plugins/nvidia-skills/README.md index e9973ad34..eb9385345 100644 --- a/plugins/nvidia-skills/README.md +++ b/plugins/nvidia-skills/README.md @@ -10,19 +10,17 @@ You can also get it by adding the self-hosted `nvidia-official` marketplace from ## What the finder runs -What it runs depends on the agent it is running in. - -**In Hermes**, it uses the built-in skills catalog and never calls `npx`: - -- **Catalog lookup.** `hermes skills search --source nvidia --json`, which reads the catalog Hermes already ships. That search needs every word in the query to match, so the finder queries one broad keyword at a time. If several keywords come back empty it reads the full catalog listing from `raw.githubusercontent.com` and picks from that, so a narrow search term can't hide a skill that exists. -- **Installs only with approval.** `hermes skills install NVIDIA/skills/`, which runs Hermes's own security scan and installs into the active profile. Hermes prompts for confirmation itself; the finder does not pass `--yes`. - -**In every other agent** (Claude Code, Codex, Cursor): - - **Catalog lookup.** The finder runs `npx skills add nvidia/skills --list` to read the live catalog, which needs Node.js and downloads the `skills` CLI from npm. If the CLI is unavailable, it reads the catalog from `github.com/NVIDIA/skills`, `build.nvidia.com/skills`, or `raw.githubusercontent.com` instead. - **Installs only with approval.** It never installs without asking. When the user agrees, it runs `npx skills add nvidia/skills --skill --global --yes`, which installs that skill from the `main` branch of `NVIDIA/skills` into the agent's global skills directory. - **Wider search.** If nothing in the NVIDIA catalog matches, it may suggest `npx skills find `, which searches skills outside NVIDIA's catalog. +### In Hermes + +Hermes ships its own scanned, profile-aware skills catalog, so the finder uses that instead and never calls `npx`. The details live in the skill's `references/hermes-catalog.md`. + +- **Catalog lookup.** `hermes skills search --source nvidia --json`. That search matches the query as a plain substring, so the finder queries one broad keyword at a time; if several come back empty it falls back to the full catalog listing on `raw.githubusercontent.com`, so a narrow search term can't hide a skill that exists. +- **Installs only with approval.** `hermes skills install NVIDIA/skills/`, which runs Hermes's own security scan and installs into the active profile. Hermes prompts for confirmation itself, so the finder does not pass `--yes`. + The plugin itself contains no code, stores no credentials, and sends no telemetry. The `skills` CLI follows its own policies. ## How this folder is built diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md index 2445b09b9..bfdb3f1eb 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md @@ -100,7 +100,6 @@ catalog check, but it is not a substitute for checking the NVIDIA skills catalog Catalog check means one of: -- `hermes skills search --source nvidia --json` (Hermes only) - `npx skills add nvidia/skills --list` - `https://github.com/NVIDIA/skills/tree/main/skills` - `https://build.nvidia.com/skills` @@ -108,33 +107,17 @@ Catalog check means one of: 1. Check whether the relevant NVIDIA skill is already installed or already in context. If it is, hand off to that skill instead of recommending install. 2. Query the live catalog before naming a specific install target or giving the - main product answer. Pick the lookup that matches the agent you are running - in. - -**Running in Hermes Agent:** use the built-in catalog, which is already scanned -and profile-aware. Do not use `npx` here. - -```bash -hermes skills search --source nvidia --json -``` - -Search one broad keyword at a time — `jetson`, `usd`, `dynamo`, `cuda` — not a -descriptive phrase, which usually returns nothing. Read the returned -descriptions and choose the match yourself. Read -[references/hermes-catalog.md](references/hermes-catalog.md) when those searches -come up empty or before installing an identifier you are unsure about. - -**Every other agent** (Claude Code, Codex, Cursor, or an unknown harness): use -the `skills` CLI. + main product answer. When shell access is available, attempt this command first: ```bash npx skills add nvidia/skills --list ``` -Do not run `hermes` commands outside Hermes. The `hermes` binary may exist on a -machine where you are running as a different agent; installing through it would -write to Hermes's skills directory, where the agent you are running in cannot -load the result. +- If running in Hermes Agent, use the Hermes CLI instead of `npx` and follow + [references/hermes-catalog.md](references/hermes-catalog.md). +- Otherwise do not run `hermes` commands. The binary may exist on the machine + even when you are a different agent, and installing through it writes to + Hermes's skills directory, where you cannot load the result. Use the fallback catalog sources only if the lookup above is unavailable, blocked, or fails: - https://github.com/NVIDIA/skills/tree/main/skills @@ -159,19 +142,7 @@ For a strong match that is not installed: The NVIDIA skill could help with this. Would you like me to install ? ``` -**Running in Hermes Agent:** install through Hermes so the skill is security -scanned and lands in the active profile. - -```bash -hermes skills install NVIDIA/skills/ -``` - -Casing is significant: `NVIDIA/...` resolves to the trusted first-party source, -while `nvidia/...` resolves to a community-trust mirror. Prefer the `Identifier` -from the search result, copied verbatim. Hermes prompts for confirmation itself, -so do not add `--yes` on the user's behalf. - -**Every other agent** — use the active agent target when known: +Use the active agent target when known: ```bash npx skills add nvidia/skills --skill --agent codex --global --yes @@ -184,6 +155,11 @@ If the agent target is unknown, omit `--agent` and let the CLI prompt: npx skills add nvidia/skills --skill --global --yes ``` +- If running in Hermes Agent, install with `hermes skills install` instead so + the skill is security scanned and lands in the active profile. Casing and + identifier rules are in + [references/hermes-catalog.md](references/hermes-catalog.md). + After install, tell the user to restart or reload the agent if their client does not pick up newly installed skills immediately. @@ -204,7 +180,7 @@ retry lookup, or continue with general help. ## When No Skills are Found Say that no strong NVIDIA catalog match was found, then either: -1. search the broader ecosystem — `hermes skills search ` without - `--source` in Hermes, otherwise `npx skills find ` — or +1. search the broader ecosystem with `npx skills find `, or in Hermes + `hermes skills search ` without `--source`, or 2. offer to help directly without a skill, or 3. suggest creating a new skill if the workflow is recurring. diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md index a23dbc3cd..486206d7f 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -1,18 +1,30 @@ -# Hermes Catalog Recovery +# Hermes Agent: Catalog Lookup and Install -Read this when `hermes skills search --source nvidia --json` returns -nothing useful, or when you are unsure whether the identifier you are about to -install is the trusted one. +Follow this instead of the `npx` commands in SKILL.md when you are running in +Hermes Agent. Hermes has a built-in catalog that is already security scanned and +profile-aware, so `npx` is both unnecessary and far more expensive in context. -## Why single keywords work and phrases do not +## Catalog lookup + +```bash +hermes skills search --source nvidia --json +``` + +Search one broad keyword at a time — `jetson`, `usd`, `dynamo`, `pandas`, +`cuda`. `--json` returns each candidate's full description; read the +descriptions and choose the match yourself. The search ranks lexically, so it +narrows the catalog but does not do the matching for you. Do not assume the +first row is the answer. + +## Why single keywords, not phrases Hermes matches a query as a plain case-insensitive substring against each entry's name, description, tags, identifier, and provider joined together. It does not tokenize, so a multi-word query only matches when those exact words appear consecutively in that order. -The practical effect is that descriptive phrases collapse to zero results while -the obvious keyword works: +Descriptive phrases therefore collapse to zero results while the obvious +keyword works: | Query | Result | | --- | --- | @@ -21,19 +33,13 @@ the obvious keyword works: | `OpenUSD` | returns only this finder | | `usd` | returns the Omniverse and USD skills | -## Search strategy - -Run two or three separate single-keyword searches rather than one phrase, and -widen by one step each time: +Run two or three separate single-keyword searches, widening by one step each +time: 1. The product name as users say it — `jetson`, `dynamo`, `cuopt`. 2. The product family — `omniverse`, `rapids`, `nemo`, `holoscan`. 3. The taxonomy lane — `usd`, `routing`, `inference`, `genomics`. -`--json` returns each candidate's full description. Read the descriptions and -choose the match yourself; do not assume the first row is the answer. The search -ranks lexically, so it narrows the catalog but does not do the matching for you. - ## Full-catalog fallback If several single-keyword searches return nothing, list the whole catalog and @@ -44,26 +50,31 @@ curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/ma | jq -r '.skills[] | "\(.name): \(.description)"' ``` -Do not use `hermes skills browse` for this. It costs more context than the full -catalog listing above and truncates both names and descriptions, so it is the -one option that is worse on both accuracy and cost. - -## Confirming the identifier before install +Do not use `hermes skills browse` for this. It costs more context than the +listing above and truncates both names and descriptions, so it is the one +option that is worse on both accuracy and cost. -Confirm the skill you picked before installing it: +## Install ```bash -hermes skills inspect +hermes skills install NVIDIA/skills/ ``` -Prefer the `Identifier` from the search result, copied verbatim, over assembling -one by hand. It may be a longer path such as `NVIDIA/skills/skills/`; -that form and the short `NVIDIA/skills/` both resolve to the same -trusted skill with the same scan hash. - Casing is what determines trust, not the number of path segments. `NVIDIA/...` resolves to the trusted first-party source; `nvidia/...` resolves to a community-trust mirror with a placeholder description. Always use `NVIDIA`. +Prefer the `Identifier` from the search result, copied verbatim, over +assembling one by hand. It may be a longer path such as +`NVIDIA/skills/skills/`; that form and the short +`NVIDIA/skills/` both resolve to the same trusted skill with the +same scan hash. + +Confirm an identifier you are unsure about before installing it: + +```bash +hermes skills inspect +``` + Hermes prompts for confirmation itself, so do not pass `--yes` on the user's behalf. diff --git a/skills/nvidia-skill-finder/SKILL.md b/skills/nvidia-skill-finder/SKILL.md index 2445b09b9..bfdb3f1eb 100644 --- a/skills/nvidia-skill-finder/SKILL.md +++ b/skills/nvidia-skill-finder/SKILL.md @@ -100,7 +100,6 @@ catalog check, but it is not a substitute for checking the NVIDIA skills catalog Catalog check means one of: -- `hermes skills search --source nvidia --json` (Hermes only) - `npx skills add nvidia/skills --list` - `https://github.com/NVIDIA/skills/tree/main/skills` - `https://build.nvidia.com/skills` @@ -108,33 +107,17 @@ Catalog check means one of: 1. Check whether the relevant NVIDIA skill is already installed or already in context. If it is, hand off to that skill instead of recommending install. 2. Query the live catalog before naming a specific install target or giving the - main product answer. Pick the lookup that matches the agent you are running - in. - -**Running in Hermes Agent:** use the built-in catalog, which is already scanned -and profile-aware. Do not use `npx` here. - -```bash -hermes skills search --source nvidia --json -``` - -Search one broad keyword at a time — `jetson`, `usd`, `dynamo`, `cuda` — not a -descriptive phrase, which usually returns nothing. Read the returned -descriptions and choose the match yourself. Read -[references/hermes-catalog.md](references/hermes-catalog.md) when those searches -come up empty or before installing an identifier you are unsure about. - -**Every other agent** (Claude Code, Codex, Cursor, or an unknown harness): use -the `skills` CLI. + main product answer. When shell access is available, attempt this command first: ```bash npx skills add nvidia/skills --list ``` -Do not run `hermes` commands outside Hermes. The `hermes` binary may exist on a -machine where you are running as a different agent; installing through it would -write to Hermes's skills directory, where the agent you are running in cannot -load the result. +- If running in Hermes Agent, use the Hermes CLI instead of `npx` and follow + [references/hermes-catalog.md](references/hermes-catalog.md). +- Otherwise do not run `hermes` commands. The binary may exist on the machine + even when you are a different agent, and installing through it writes to + Hermes's skills directory, where you cannot load the result. Use the fallback catalog sources only if the lookup above is unavailable, blocked, or fails: - https://github.com/NVIDIA/skills/tree/main/skills @@ -159,19 +142,7 @@ For a strong match that is not installed: The NVIDIA skill could help with this. Would you like me to install ? ``` -**Running in Hermes Agent:** install through Hermes so the skill is security -scanned and lands in the active profile. - -```bash -hermes skills install NVIDIA/skills/ -``` - -Casing is significant: `NVIDIA/...` resolves to the trusted first-party source, -while `nvidia/...` resolves to a community-trust mirror. Prefer the `Identifier` -from the search result, copied verbatim. Hermes prompts for confirmation itself, -so do not add `--yes` on the user's behalf. - -**Every other agent** — use the active agent target when known: +Use the active agent target when known: ```bash npx skills add nvidia/skills --skill --agent codex --global --yes @@ -184,6 +155,11 @@ If the agent target is unknown, omit `--agent` and let the CLI prompt: npx skills add nvidia/skills --skill --global --yes ``` +- If running in Hermes Agent, install with `hermes skills install` instead so + the skill is security scanned and lands in the active profile. Casing and + identifier rules are in + [references/hermes-catalog.md](references/hermes-catalog.md). + After install, tell the user to restart or reload the agent if their client does not pick up newly installed skills immediately. @@ -204,7 +180,7 @@ retry lookup, or continue with general help. ## When No Skills are Found Say that no strong NVIDIA catalog match was found, then either: -1. search the broader ecosystem — `hermes skills search ` without - `--source` in Hermes, otherwise `npx skills find ` — or +1. search the broader ecosystem with `npx skills find `, or in Hermes + `hermes skills search ` without `--source`, or 2. offer to help directly without a skill, or 3. suggest creating a new skill if the workflow is recurring. diff --git a/skills/nvidia-skill-finder/references/hermes-catalog.md b/skills/nvidia-skill-finder/references/hermes-catalog.md index a23dbc3cd..486206d7f 100644 --- a/skills/nvidia-skill-finder/references/hermes-catalog.md +++ b/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -1,18 +1,30 @@ -# Hermes Catalog Recovery +# Hermes Agent: Catalog Lookup and Install -Read this when `hermes skills search --source nvidia --json` returns -nothing useful, or when you are unsure whether the identifier you are about to -install is the trusted one. +Follow this instead of the `npx` commands in SKILL.md when you are running in +Hermes Agent. Hermes has a built-in catalog that is already security scanned and +profile-aware, so `npx` is both unnecessary and far more expensive in context. -## Why single keywords work and phrases do not +## Catalog lookup + +```bash +hermes skills search --source nvidia --json +``` + +Search one broad keyword at a time — `jetson`, `usd`, `dynamo`, `pandas`, +`cuda`. `--json` returns each candidate's full description; read the +descriptions and choose the match yourself. The search ranks lexically, so it +narrows the catalog but does not do the matching for you. Do not assume the +first row is the answer. + +## Why single keywords, not phrases Hermes matches a query as a plain case-insensitive substring against each entry's name, description, tags, identifier, and provider joined together. It does not tokenize, so a multi-word query only matches when those exact words appear consecutively in that order. -The practical effect is that descriptive phrases collapse to zero results while -the obvious keyword works: +Descriptive phrases therefore collapse to zero results while the obvious +keyword works: | Query | Result | | --- | --- | @@ -21,19 +33,13 @@ the obvious keyword works: | `OpenUSD` | returns only this finder | | `usd` | returns the Omniverse and USD skills | -## Search strategy - -Run two or three separate single-keyword searches rather than one phrase, and -widen by one step each time: +Run two or three separate single-keyword searches, widening by one step each +time: 1. The product name as users say it — `jetson`, `dynamo`, `cuopt`. 2. The product family — `omniverse`, `rapids`, `nemo`, `holoscan`. 3. The taxonomy lane — `usd`, `routing`, `inference`, `genomics`. -`--json` returns each candidate's full description. Read the descriptions and -choose the match yourself; do not assume the first row is the answer. The search -ranks lexically, so it narrows the catalog but does not do the matching for you. - ## Full-catalog fallback If several single-keyword searches return nothing, list the whole catalog and @@ -44,26 +50,31 @@ curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/ma | jq -r '.skills[] | "\(.name): \(.description)"' ``` -Do not use `hermes skills browse` for this. It costs more context than the full -catalog listing above and truncates both names and descriptions, so it is the -one option that is worse on both accuracy and cost. - -## Confirming the identifier before install +Do not use `hermes skills browse` for this. It costs more context than the +listing above and truncates both names and descriptions, so it is the one +option that is worse on both accuracy and cost. -Confirm the skill you picked before installing it: +## Install ```bash -hermes skills inspect +hermes skills install NVIDIA/skills/ ``` -Prefer the `Identifier` from the search result, copied verbatim, over assembling -one by hand. It may be a longer path such as `NVIDIA/skills/skills/`; -that form and the short `NVIDIA/skills/` both resolve to the same -trusted skill with the same scan hash. - Casing is what determines trust, not the number of path segments. `NVIDIA/...` resolves to the trusted first-party source; `nvidia/...` resolves to a community-trust mirror with a placeholder description. Always use `NVIDIA`. +Prefer the `Identifier` from the search result, copied verbatim, over +assembling one by hand. It may be a longer path such as +`NVIDIA/skills/skills/`; that form and the short +`NVIDIA/skills/` both resolve to the same trusted skill with the +same scan hash. + +Confirm an identifier you are unsure about before installing it: + +```bash +hermes skills inspect +``` + Hermes prompts for confirmation itself, so do not pass `--yes` on the user's behalf. From 503a2f9a13b3c571b1d062946dd302b4302dc3e9 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Wed, 4 Nov 2026 11:29:01 -0500 Subject: [PATCH 05/19] revert(nvidia-skill-finder): restore the original wider-search step Keep 'When No Skills are Found' exactly as it was on main; the Hermes variant of the ecosystem search does not need to be called out there. Signed-off-by: Jason Dudash --- plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md | 3 +-- skills/nvidia-skill-finder/SKILL.md | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md index bfdb3f1eb..4bea2e239 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md @@ -180,7 +180,6 @@ retry lookup, or continue with general help. ## When No Skills are Found Say that no strong NVIDIA catalog match was found, then either: -1. search the broader ecosystem with `npx skills find `, or in Hermes - `hermes skills search ` without `--source`, or +1. search the broader ecosystem with npx skills find , or 2. offer to help directly without a skill, or 3. suggest creating a new skill if the workflow is recurring. diff --git a/skills/nvidia-skill-finder/SKILL.md b/skills/nvidia-skill-finder/SKILL.md index bfdb3f1eb..4bea2e239 100644 --- a/skills/nvidia-skill-finder/SKILL.md +++ b/skills/nvidia-skill-finder/SKILL.md @@ -180,7 +180,6 @@ retry lookup, or continue with general help. ## When No Skills are Found Say that no strong NVIDIA catalog match was found, then either: -1. search the broader ecosystem with `npx skills find `, or in Hermes - `hermes skills search ` without `--source`, or +1. search the broader ecosystem with npx skills find , or 2. offer to help directly without a skill, or 3. suggest creating a new skill if the workflow is recurring. From c3e321f7b475b97c7b8ec0f8c870d82d369ed514 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Wed, 4 Nov 2026 13:31:41 -0500 Subject: [PATCH 06/19] chore(versions): refresh nvidia-skill-finder entry generate_versions.py tracks the last commit touching each skill directory, so editing the skill leaves versions.json stale. Signed-off-by: Jason Dudash --- versions.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions.json b/versions.json index 3f6809531..07481015d 100644 --- a/versions.json +++ b/versions.json @@ -2119,8 +2119,8 @@ "name": "nvidia-skill-finder", "path": "skills/nvidia-skill-finder", "content_digest": "sha256:da43951c0c8c69da5622f5299055228c70bc6345273e7869f3c59acb92af8310", - "last_commit": "13a0ce1d97a350e2a7c6d0ca82fea37d293af625", - "last_modified": "2026-10-01" + "last_commit": "dce10b59d664f5939c58602cadc4db28fab768be", + "last_modified": "2026-11-04" }, { "name": "nvshmem-collect-performance-data", From 6ac9a59ca757cde60d38c7206da3b121fc0258ff Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Wed, 4 Nov 2026 20:35:21 +0000 Subject: [PATCH 07/19] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- .../skills/nvidia-skill-finder/BENCHMARK.md | 144 +++++++++++------- .../skills/nvidia-skill-finder/skill-card.md | 87 +++++++---- .../skills/nvidia-skill-finder/skill.oms.sig | 2 +- skills/nvidia-skill-finder/BENCHMARK.md | 142 ++++++++++------- skills/nvidia-skill-finder/skill-card.md | 94 ++++++++---- skills/nvidia-skill-finder/skill.oms.sig | 2 +- 6 files changed, 296 insertions(+), 175 deletions(-) diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md index 79fd0d79b..0337d86f8 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md @@ -9,18 +9,33 @@ Recommended for publication based on the completed evaluation evidence in this r ## Evaluation Metadata - Skill: `nvidia-skill-finder` -- Evaluation date: 2026-10-01 -- Evaluator version: `1.5.6` -- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Validation status: `passed` +- Evaluation date: 2026-11-04 +- Evaluator version: `1.6.6` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 18 evaluation tasks (13 positive, 5 negative) -- Dataset digest: `sha256:df412e229dfed3687e1112c8f5bc439a6f3695291f3d357f049002621e90845f` (skill-evaluator-dataset-snapshot/1) -- Attempts per task: 3 +- Dataset digest: `sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 - Environment: `k8s-sandbox` - Tier 2 evidence: required for publication - Tier 3 evidence: required for publication Each task attempt ran in its own isolated sandbox pod. +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| nvidia-skill-finder | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | + +- Controls for `nvidia-skill-finder`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + ## What This Report Answers The three-tier evaluation checks whether the skill: @@ -35,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 98.2% → 97.2% (-1.0 points) | 88.5% → 94.4% (+5.9 points) | -| Correctness | 69.6% → 96.7% (+27.1 points) | 70.8% → 100.0% (+29.2 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 36.1% → 87.9% (+51.8 points) | 40.9% → 89.7% (+48.8 points) | -| Efficiency | 78.4% — baseline ran, but no comparable score was available; uplift unavailable | 66.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 94.1% — baseline ran, but no comparable score was available; uplift unavailable | 87.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 97.2% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 62.2% → 96.7% (+34.5 points) | 77.8% → 91.1% (+13.3 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 39.9% → 96.5% (+56.6 points) | 59.3% → 92.1% (+32.8 points) | +| Efficiency | 80.3% — baseline ran, but no comparable score was available; uplift unavailable | 61.3% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -48,59 +63,70 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 96.4% (-1.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 90.0% → 88.9% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + ## Token Usage Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 2,931,632 | 4,494,280 | N/A | N/A | skill 18/18; base 27/27 | -| claude-code | nvidia-skill-finder-diff-bare-invocation | 175,571 | 151,559 | +24,012 | +15.84% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 136,133 | 120,437 | +15,696 | +13.03% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-deep-research | 170,078 | 363,184 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-neg-express-route | 356,710 | 483,643 | -126,933 | -26.25% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 154,997 | 185,511 | -30,514 | -16.45% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-python-refactor | 120,042 | 119,199 | +843 | +0.71% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-react-optimize | 153,960 | 152,401 | +1,559 | +1.02% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-video-editing | 159,058 | 154,957 | +4,101 | +2.65% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-cad-to-simready | 129,524 | 162,375 | -32,851 | -20.23% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dicom | 177,291 | 157,064 | +20,227 | +12.88% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dynamo-kv | 129,903 | 376,086 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-gpu-pandas | 130,132 | 124,405 | +5,727 | +4.60% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-isaac-lab | 211,676 | 60,437 | +151,239 | +250.24% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 167,216 | 59,583 | N/A | N/A | skill 1/1; base 2/2 | -| claude-code | nvidia-skill-finder-pos-jetson-setup | 130,190 | 335,895 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-multigpu-training | 169,065 | 1,276,278 | -1,107,213 | -86.75% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-openusd-optimization | 129,543 | 89,238 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-vehicle-routing | 130,543 | 122,028 | +8,515 | +6.98% | skill 1/1; base 1/1 | -| codex | All cases | 1,642,585 | 2,128,791 | N/A | N/A | skill 18/18; base 26/26 | -| codex | nvidia-skill-finder-diff-bare-invocation | 73,710 | 685,152 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-diff-browse-no-keywords | 128,756 | 240,494 | -111,738 | -46.46% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-deep-research | 86,599 | 40,331 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-neg-express-route | 177,679 | 69,297 | +108,382 | +156.40% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-generic-kubernetes | 201,213 | 213,286 | -12,073 | -5.66% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-python-refactor | 41,853 | 69,141 | -27,288 | -39.47% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-react-optimize | 98,985 | 127,238 | -28,253 | -22.20% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-video-editing | 69,541 | 55,477 | +14,064 | +25.35% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-cad-to-simready | 74,083 | 25,338 | +48,745 | +192.38% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dicom | 74,117 | 31,845 | +42,272 | +132.74% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dynamo-kv | 94,291 | 307,770 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-pos-gpu-pandas | 71,906 | 18,127 | +53,779 | +296.68% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-isaac-lab | 85,260 | 74,373 | +10,887 | +14.64% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-driver-install | 71,998 | 18,014 | +53,984 | +299.68% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-setup | 71,193 | 66,628 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-pos-multigpu-training | 74,250 | 59,276 | +14,974 | +25.26% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-openusd-optimization | 79,947 | 13,501 | +66,446 | +492.16% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-vehicle-routing | 67,204 | 13,503 | +53,701 | +397.70% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 4,574,217 | 6,623,071 | N/A | N/A | skill 36/36; base 53/53 | - -Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. +| claude-code | All cases | 2,746,750 | 1,692,842 | +1,053,908 | +62.26% | skill 18/18; base 18/18 | +| claude-code | nvidia-skill-finder-diff-bare-invocation | 158,485 | 70,559 | +87,926 | +124.61% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 160,553 | 107,298 | +53,255 | +49.63% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-deep-research | 153,825 | 35,323 | +118,502 | +335.48% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-express-route | 224,006 | 182,756 | +41,250 | +22.57% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 109,046 | 142,661 | -33,615 | -23.56% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-python-refactor | 107,662 | 106,575 | +1,087 | +1.02% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-react-optimize | 144,923 | 107,108 | +37,815 | +35.31% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-video-editing | 108,246 | 71,223 | +37,023 | +51.98% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-cad-to-simready | 153,252 | 143,560 | +9,692 | +6.75% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dicom | 154,197 | 35,935 | +118,262 | +329.10% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dynamo-kv | 195,216 | 144,272 | +50,944 | +35.31% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,152 | 107,398 | +45,754 | +42.60% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,942 | 36,723 | +118,219 | +321.92% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,508 | 35,926 | +117,582 | +327.29% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,903 | 107,200 | +46,703 | +43.57% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,782 | 151,902 | +2,880 | +1.90% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,925 | 70,956 | +82,969 | +116.93% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,127 | 35,467 | +117,660 | +331.75% | skill 1/1; base 1/1 | +| codex | All cases | 1,130,804 | 610,948 | +519,856 | +85.09% | skill 18/18; base 18/18 | +| codex | nvidia-skill-finder-diff-bare-invocation | 68,890 | 35,919 | +32,971 | +91.79% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-browse-no-keywords | 89,599 | 52,784 | +36,815 | +69.75% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-deep-research | 90,172 | 35,866 | +54,306 | +151.41% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-express-route | 55,483 | 40,948 | +14,535 | +35.50% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,915 | 27,157 | +14,758 | +54.34% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-python-refactor | 41,302 | 40,437 | +865 | +2.14% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-react-optimize | 40,956 | 41,097 | -141 | -0.34% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-video-editing | 13,505 | 13,316 | +189 | +1.42% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-cad-to-simready | 67,881 | 13,365 | +54,516 | +407.90% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dicom | 73,056 | 38,531 | +34,525 | +89.60% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dynamo-kv | 17,983 | 31,520 | -13,537 | -42.95% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-gpu-pandas | 69,058 | 22,021 | +47,037 | +213.60% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-isaac-lab | 106,097 | 56,158 | +49,939 | +88.93% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-driver-install | 69,157 | 17,804 | +51,353 | +288.44% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-setup | 69,085 | 17,655 | +51,430 | +291.31% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-multigpu-training | 77,888 | 22,466 | +55,422 | +246.69% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-openusd-optimization | 68,381 | 65,091 | +3,290 | +5.05% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-vehicle-routing | 70,396 | 38,813 | +31,583 | +81.37% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 3,877,554 | 2,303,790 | +1,573,764 | +68.31% | skill 36/36; base 36/36 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. ## Tier Status | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 19 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | @@ -109,12 +135,12 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c
Show detailed findings and successful checks -- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nvidia-skill-finder/SKILL.md`) -- **MEDIUM** SECURITY/Unknown (SQP-1): L032 instructs activation on any "how do I do X" request where X "might be a common task that an NVIDIA skill can help w (`SKILL.md:32`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: The `npx skills add nvidia/skills --list` command is invoked without a pinned package version. When npx resolves an unpi (`SKILL.md:74`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: Same unpinned `npx skills` invocation as above, used as the primary catalog-check mechanism. Without a version pin, any (`SKILL.md:103`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: Another instance of the same unpinned `npx skills` pattern. The cumulative effect of multiple unpinned invocations acros (`SKILL.md:113`) -- 14 additional finding(s) are available in the full evaluation artifacts. +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:74`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:135`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:103`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:148`) +- 9 additional finding(s) are available in the full evaluation artifacts.
diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md index b5487a7ed..cbaebadd5 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Helps users discover, install, and start using NVIDIA agent skills by searching the live NVIDIA skills catalog and recommending matches for their task.
+Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
CC-BY-4.0 AND Apache-2.0
## Use Case:
-Developers and engineers use this skill to find and install NVIDIA-verified agent skills that match their task, covering domains such as Physical AI, Robotics, Vision AI, Data Science, Inference, Training, Decision Optimization, and GPU Development.
+Developers and engineers working with NVIDIA products, hardware, SDKs, and services use this skill to discover, recommend, and (with explicit approval) install the most relevant NVIDIA agent skill from the live NVIDIA skills catalog for their task.
### Deployment Geography for Use:
Global
@@ -25,55 +25,90 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [NVIDIA Taxonomy Routing](references/taxonomy-routing.md)
-- [NVIDIA Skills Catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
-- [NVIDIA Skills Catalog (Build Portal)](https://build.nvidia.com/skills)
-- [NVIDIA Agent Skills Documentation](https://docs.nvidia.com/skills)
+- [references/taxonomy-routing.md](references/taxonomy-routing.md)
+- [references/hermes-catalog.md](references/hermes-catalog.md)
+- [NVIDIA Skills on GitHub](https://github.com/NVIDIA/skills/tree/main/skills)
+- [NVIDIA Skills on build.nvidia.com](https://build.nvidia.com/skills)
+- [NVIDIA Skills catalog index (skills.sh.json)](https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json)
+- [NVIDIA Skills documentation](https://docs.nvidia.com/skills)
## Skill Output:
-**Output Type(s):** [Analysis, Shell commands, Configuration instructions]
+**Output Type(s):** [Analysis, Shell commands]
**Output Format:** [Markdown with inline bash code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [None]
+**Other Properties Related to Output:** [Recommends at most three catalog skills ordered by confidence; install commands are proposed only and run only after explicit user approval.]
## Evaluation Agents Used:
-- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
-- Codex (`openai/openai/gpt-5.5`)
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
## Evaluation Tasks:
-18 evaluation tasks (13 positive, 5 negative), each run with 3 attempts per task in isolated k8s-sandbox pods.
+18 evaluation tasks (13 positive, 5 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`), compared against a no-skill baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
-- Correctness: Checks final-answer correctness against the reference answer.
-- Discoverability: Checks whether the expected skill was selected and the workflow executed.
-- Effectiveness: Checks whether the user's goal was achieved and the expected workflow behavior was followed.
-- Efficiency: Checks tool-call productivity and token usage efficiency.
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
- `security`: Unsafe operations, secret leakage, and unauthorized access.
- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
- `accuracy`: Final-answer correctness against the reference answer.
- `goal_accuracy`: Whether the user's goal was achieved.
- `behavior_check`: Whether the expected workflow behavior was followed.
-- `skill_efficiency`: Tool-call productivity (routing scored under Discoverability).
-- `token_efficiency`: Actual uncached prompt plus completion token usage.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
-| Dimension | Tasks | Claude Code | Codex | -|---|---:|---:|---:| -| Overall | 18 | 92.0% | 89.0% | -| Security | 18 | 97.2% | 94.4% | -| Correctness | 18 | 96.7% | 100.0% | -| Discoverability | 18 | 100.0% | 94.6% | -| Effectiveness | 18 | 87.9% | 89.7% | -| Efficiency | 18 | 78.4% | 66.5% | +> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 94.1% — baseline ran, but no comparable score was available; uplift unavailable | 87.1% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 97.2% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 62.2% → 96.7% (+34.5 points) | 77.8% → 91.1% (+13.3 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 39.9% → 96.5% (+56.6 points) | 59.3% → 92.1% (+32.8 points) | +| Efficiency | 80.3% — baseline ran, but no comparable score was available; uplift unavailable | 61.3% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 96.4% (-1.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 90.0% → 88.9% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | ## Skill Version(s):
0.3.0 (source: changelog, released 2025-05-21)
diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig index cdde023c3..b4d8ad419 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIwNTM4NDcwMjkyNDZlYjI1NTA2YzMyZjk0MDQ5NDJlOGMyNzM2MzdiYjA1ODI5MzhlMjAwYTU5ZGQ5Mzg0YmE1IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiZWJjN2RlMTc5OGIyYmJhZTE1ZjExOGMyNWVjZjQwOThkYThlN2Q4MmVhYjJhMjhjZWEyNTgwNTA1ZjQwZTBjOSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJmNTllNWMyYzllODljZDM1NjM3NjQyMjUyODBmOWE4ZjA1ZWExMTY0ZWZhOTc4OGZlYTU4OTJmNjViMDRkZjJjIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFnZW50cy9vcGVuYWkueWFtbCIsCiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiNzE2ZmNiM2ZiMTFkMDM2MjYyYTY5NjY0OTI4MmZlYTc2ZmE2MGQyNDIzYTc4MDFlMWQ0YTNlOGY3OTMzZjBlNCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3RheG9ub215LXJvdXRpbmcubWQiLAogICAgICAgICJkaWdlc3QiOiAiYTI0YjU5NGIwNmZhNGUxNmM4OTM3ZGE1NGI5YmZjMmNiZWNlOGJjYzY2YjlkMDJhNjcyOTQzNDAyMWExNjg0OCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjI5OWRkZjA3NzUzOTU3ODY2MzQxMzQ5YjQ5N2MxODQ4MWNmYzI1NWE5MTMxZWYzNTAyNDM4Y2Y3NDA3MTg3OTAiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMGGg1+VbtPZ6EXlNDc0xHBMjfZN0EAGw1oDeoYkA/BuRWHpci0EFllec/wl0PyaTswIxAJLsjn6E8DOrgQvHZgdt+8ILmYrhydTiQ6wLzhXUeJKe3vGsgSFDv0BLA1hZ1kp1ig==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIyNjQ3MzJjOGVmZDk5MDJkMTI5MjA3NTlkZTkwMjg0YWYyYmYxNDdlZGNkNjg1YTIyY2IzYTk4ZTcyODhjNzY0IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjVkODcyMDgwZDg0YmYxZDU2OWNlOGM3MjFjZjRlZGM4ZWI2YTk1ZjBhMjIyYTUwOWQwMzExYWI4NDdmYjgxMmEiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjc0NDJjNWEwNzUxNjU3N2FiYTdlOThkNjI5MjMxNjJiOGFmNmZjNTkzZDZmZmJhZTFkZjMxY2Q1NjY5N2Y0ZmUiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZjJiMjk0MjI1MjA2MTI1MDU2NDNjNmExZGQyZjQ0MWI0YTFiZjM1MGZhNTFjNmZlNmVmNDVhNTI2YTcyYjczOSIsCiAgICAgICAgIm5hbWUiOiAiYWdlbnRzL29wZW5haS55YW1sIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiM2U0MGI3NGYzNzFmZDMwMzAyMzgyNzAxNDFhNjdhYjJjNjhhMzUyYzgzZmM3MTZkZDk0NDI3MDgwNzNjNDc1YSIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjExYzRjYTllMTFmYzM2YzA0MGIxZTI4MGRkZGJjYmU4OWRiNGZhMTEyMDhiNmNlMzY1OGIyZWM5MjYxNjdlNGYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvaGVybWVzLWNhdGFsb2cubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJhMjRiNTk0YjA2ZmE0ZTE2Yzg5MzdkYTU0YjliZmMyY2JlY2U4YmNjNjZiOWQwMmE2NzI5NDM0MDIxYTE2ODQ4IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3RheG9ub215LXJvdXRpbmcubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIyNzc1MjVlZTExNzQ5ZTAzZGZjYjc1MDVmZDI0Y2ExZjJiYjQ1NzIyNDZmODU1M2FiMWUxMWZhZmZmZGFjNjk0IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMGgUGS6iicxQ8OlZHs2w00O27KHwotieqogR2IHsISSLIxw9dQ5WL64Af7nez+UFdAIxAK6S4T1WMwigGm4XHTlZpKIbFCO5Qo8ISCV2SLwsrEXHcvKRtdRX3KZHCFQY2dxytQ==","keyid":""}]}} \ No newline at end of file diff --git a/skills/nvidia-skill-finder/BENCHMARK.md b/skills/nvidia-skill-finder/BENCHMARK.md index 79fd0d79b..263e29911 100644 --- a/skills/nvidia-skill-finder/BENCHMARK.md +++ b/skills/nvidia-skill-finder/BENCHMARK.md @@ -9,18 +9,33 @@ Recommended for publication based on the completed evaluation evidence in this r ## Evaluation Metadata - Skill: `nvidia-skill-finder` -- Evaluation date: 2026-10-01 -- Evaluator version: `1.5.6` -- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Validation status: `passed` +- Evaluation date: 2026-11-04 +- Evaluator version: `1.6.6` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 18 evaluation tasks (13 positive, 5 negative) -- Dataset digest: `sha256:df412e229dfed3687e1112c8f5bc439a6f3695291f3d357f049002621e90845f` (skill-evaluator-dataset-snapshot/1) -- Attempts per task: 3 +- Dataset digest: `sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 - Environment: `k8s-sandbox` - Tier 2 evidence: required for publication - Tier 3 evidence: required for publication Each task attempt ran in its own isolated sandbox pod. +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| nvidia-skill-finder | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | + +- Controls for `nvidia-skill-finder`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + ## What This Report Answers The three-tier evaluation checks whether the skill: @@ -35,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 98.2% → 97.2% (-1.0 points) | 88.5% → 94.4% (+5.9 points) | -| Correctness | 69.6% → 96.7% (+27.1 points) | 70.8% → 100.0% (+29.2 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 36.1% → 87.9% (+51.8 points) | 40.9% → 89.7% (+48.8 points) | -| Efficiency | 78.4% — baseline ran, but no comparable score was available; uplift unavailable | 66.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 97.2% (-2.8 points) | 97.2% → 91.7% (-5.5 points) | +| Correctness | 64.4% → 96.7% (+32.3 points) | 77.8% → 91.1% (+13.3 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 37.1% → 93.8% (+56.7 points) | 58.8% → 94.2% (+35.4 points) | +| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 64.7% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -48,59 +63,70 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 96.2% (-2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 89.3% → 88.2% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + ## Token Usage Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 2,931,632 | 4,494,280 | N/A | N/A | skill 18/18; base 27/27 | -| claude-code | nvidia-skill-finder-diff-bare-invocation | 175,571 | 151,559 | +24,012 | +15.84% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 136,133 | 120,437 | +15,696 | +13.03% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-deep-research | 170,078 | 363,184 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-neg-express-route | 356,710 | 483,643 | -126,933 | -26.25% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 154,997 | 185,511 | -30,514 | -16.45% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-python-refactor | 120,042 | 119,199 | +843 | +0.71% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-react-optimize | 153,960 | 152,401 | +1,559 | +1.02% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-video-editing | 159,058 | 154,957 | +4,101 | +2.65% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-cad-to-simready | 129,524 | 162,375 | -32,851 | -20.23% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dicom | 177,291 | 157,064 | +20,227 | +12.88% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dynamo-kv | 129,903 | 376,086 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-gpu-pandas | 130,132 | 124,405 | +5,727 | +4.60% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-isaac-lab | 211,676 | 60,437 | +151,239 | +250.24% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 167,216 | 59,583 | N/A | N/A | skill 1/1; base 2/2 | -| claude-code | nvidia-skill-finder-pos-jetson-setup | 130,190 | 335,895 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-multigpu-training | 169,065 | 1,276,278 | -1,107,213 | -86.75% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-openusd-optimization | 129,543 | 89,238 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-vehicle-routing | 130,543 | 122,028 | +8,515 | +6.98% | skill 1/1; base 1/1 | -| codex | All cases | 1,642,585 | 2,128,791 | N/A | N/A | skill 18/18; base 26/26 | -| codex | nvidia-skill-finder-diff-bare-invocation | 73,710 | 685,152 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-diff-browse-no-keywords | 128,756 | 240,494 | -111,738 | -46.46% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-deep-research | 86,599 | 40,331 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-neg-express-route | 177,679 | 69,297 | +108,382 | +156.40% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-generic-kubernetes | 201,213 | 213,286 | -12,073 | -5.66% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-python-refactor | 41,853 | 69,141 | -27,288 | -39.47% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-react-optimize | 98,985 | 127,238 | -28,253 | -22.20% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-video-editing | 69,541 | 55,477 | +14,064 | +25.35% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-cad-to-simready | 74,083 | 25,338 | +48,745 | +192.38% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dicom | 74,117 | 31,845 | +42,272 | +132.74% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dynamo-kv | 94,291 | 307,770 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-pos-gpu-pandas | 71,906 | 18,127 | +53,779 | +296.68% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-isaac-lab | 85,260 | 74,373 | +10,887 | +14.64% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-driver-install | 71,998 | 18,014 | +53,984 | +299.68% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-setup | 71,193 | 66,628 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-pos-multigpu-training | 74,250 | 59,276 | +14,974 | +25.26% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-openusd-optimization | 79,947 | 13,501 | +66,446 | +492.16% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-vehicle-routing | 67,204 | 13,503 | +53,701 | +397.70% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 4,574,217 | 6,623,071 | N/A | N/A | skill 36/36; base 53/53 | - -Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. +| claude-code | All cases | 2,940,059 | 1,733,228 | +1,206,831 | +69.63% | skill 18/18; base 18/18 | +| claude-code | nvidia-skill-finder-diff-bare-invocation | 159,556 | 107,051 | +52,505 | +49.05% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 160,640 | 106,788 | +53,852 | +50.43% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-deep-research | 349,223 | 35,273 | +313,950 | +890.06% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-express-route | 185,632 | 223,103 | -37,471 | -16.80% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 72,345 | 71,645 | +700 | +0.98% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-python-refactor | 142,972 | 141,956 | +1,016 | +0.72% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-react-optimize | 145,032 | 142,851 | +2,181 | +1.53% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-video-editing | 107,845 | 71,165 | +36,680 | +51.54% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-cad-to-simready | 154,290 | 143,642 | +10,648 | +7.41% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dicom | 154,191 | 35,577 | +118,614 | +333.40% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dynamo-kv | 231,606 | 144,237 | +87,369 | +60.57% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,338 | 108,293 | +45,045 | +41.60% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,754 | 72,321 | +82,433 | +113.98% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,792 | 35,680 | +118,112 | +331.03% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,387 | 71,488 | +81,899 | +114.56% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,938 | 151,301 | +3,637 | +2.40% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,414 | 35,400 | +118,014 | +333.37% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,104 | 35,457 | +117,647 | +331.80% | skill 1/1; base 1/1 | +| codex | All cases | 1,208,315 | 806,320 | +401,995 | +49.86% | skill 18/18; base 18/18 | +| codex | nvidia-skill-finder-diff-bare-invocation | 86,372 | 35,913 | +50,459 | +140.50% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-browse-no-keywords | 93,587 | 47,277 | +46,310 | +97.95% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-deep-research | 80,712 | 187,624 | -106,912 | -56.98% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-express-route | 40,930 | 40,522 | +408 | +1.01% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,679 | 27,181 | +14,498 | +53.34% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-python-refactor | 40,995 | 40,331 | +664 | +1.65% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-react-optimize | 69,618 | 40,882 | +28,736 | +70.29% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-video-editing | 13,444 | 13,257 | +187 | +1.41% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-cad-to-simready | 67,899 | 13,327 | +54,572 | +409.48% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dicom | 126,618 | 57,984 | +68,634 | +118.37% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dynamo-kv | 71,047 | 17,627 | +53,420 | +303.06% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-gpu-pandas | 53,320 | 26,172 | +27,148 | +103.73% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-isaac-lab | 103,549 | 52,434 | +51,115 | +97.48% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-driver-install | 61,973 | 17,645 | +44,328 | +251.22% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-setup | 53,437 | 17,828 | +35,609 | +199.74% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-multigpu-training | 78,115 | 66,339 | +11,776 | +17.75% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-openusd-optimization | 54,639 | 65,106 | -10,467 | -16.08% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-vehicle-routing | 70,381 | 38,871 | +31,510 | +81.06% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 4,148,374 | 2,539,548 | +1,608,826 | +63.35% | skill 36/36; base 36/36 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. ## Tier Status | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 19 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | @@ -110,11 +136,11 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c Show detailed findings and successful checks - **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nvidia-skill-finder/SKILL.md`) -- **MEDIUM** SECURITY/Unknown (SQP-1): L032 instructs activation on any "how do I do X" request where X "might be a common task that an NVIDIA skill can help w (`SKILL.md:32`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: The `npx skills add nvidia/skills --list` command is invoked without a pinned package version. When npx resolves an unpi (`SKILL.md:74`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: Same unpinned `npx skills` invocation as above, used as the primary catalog-check mechanism. Without a version pin, any (`SKILL.md:103`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: Another instance of the same unpinned `npx skills` pattern. The cumulative effect of multiple unpinned invocations acros (`SKILL.md:113`) -- 14 additional finding(s) are available in the full evaluation artifacts. +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:74`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:103`) +- **MEDIUM** SECURITY/Security finding (SQP-2): Because the skill triggers on many NVIDIA-related requests without being asked, the agent may download and run remote np (`SKILL.md:109`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:113`) +- 11 additional finding(s) are available in the full evaluation artifacts. diff --git a/skills/nvidia-skill-finder/skill-card.md b/skills/nvidia-skill-finder/skill-card.md index b5487a7ed..4e8547b49 100644 --- a/skills/nvidia-skill-finder/skill-card.md +++ b/skills/nvidia-skill-finder/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Helps users discover, install, and start using NVIDIA agent skills by searching the live NVIDIA skills catalog and recommending matches for their task.
+Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill; it checks the live NVIDIA skills catalog and recommends matching skills, while staying quiet for generic non-NVIDIA route, optimize, deploy, AI, video, data, or infrastructure tasks.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
CC-BY-4.0 AND Apache-2.0
## Use Case:
-Developers and engineers use this skill to find and install NVIDIA-verified agent skills that match their task, covering domains such as Physical AI, Robotics, Vision AI, Data Science, Inference, Training, Decision Optimization, and GPU Development.
+Developers and engineers working with NVIDIA products, hardware, SDKs, and services use this skill to discover, evaluate, and (with explicit approval) install relevant NVIDIA agent skills from the live catalog, including Hermes Agent catalog lookups.
### Deployment Geography for Use:
Global
@@ -25,55 +25,89 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
+- [Hermes Agent: Catalog Lookup and Install](references/hermes-catalog.md)
- [NVIDIA Taxonomy Routing](references/taxonomy-routing.md)
-- [NVIDIA Skills Catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
-- [NVIDIA Skills Catalog (Build Portal)](https://build.nvidia.com/skills)
-- [NVIDIA Agent Skills Documentation](https://docs.nvidia.com/skills)
+- [NVIDIA Skills catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
+- [NVIDIA Skills on build.nvidia.com](https://build.nvidia.com/skills)
+- [NVIDIA Skills catalog index (skills.sh.json)](https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json)
## Skill Output:
-**Output Type(s):** [Analysis, Shell commands, Configuration instructions]
+**Output Type(s):** [Analysis, Shell commands]
**Output Format:** [Markdown with inline bash code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [None]
+**Other Properties Related to Output:** [Recommends at most three skills ordered by confidence; installs only after explicit user approval.]
## Evaluation Agents Used:
-- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
-- Codex (`openai/openai/gpt-5.5`)
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
## Evaluation Tasks:
-18 evaluation tasks (13 positive, 5 negative), each run with 3 attempts per task in isolated k8s-sandbox pods.
+18 evaluation tasks (13 positive, 5 negative), 1 attempt per task, run in the `k8s-sandbox` environment with each task attempt in its own isolated sandbox pod; each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
-- Correctness: Checks final-answer correctness against the reference answer.
-- Discoverability: Checks whether the expected skill was selected and the workflow executed.
-- Effectiveness: Checks whether the user's goal was achieved and the expected workflow behavior was followed.
-- Efficiency: Checks tool-call productivity and token usage efficiency.
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `accuracy`: Final-answer correctness against the reference answer.
-- `goal_accuracy`: Whether the user's goal was achieved.
-- `behavior_check`: Whether the expected workflow behavior was followed.
-- `skill_efficiency`: Tool-call productivity (routing scored under Discoverability).
-- `token_efficiency`: Actual uncached prompt plus completion token usage.
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
-| Dimension | Tasks | Claude Code | Codex | -|---|---:|---:|---:| -| Overall | 18 | 92.0% | 89.0% | -| Security | 18 | 97.2% | 94.4% | -| Correctness | 18 | 96.7% | 100.0% | -| Discoverability | 18 | 100.0% | 94.6% | -| Effectiveness | 18 | 87.9% | 89.7% | -| Efficiency | 18 | 78.4% | 66.5% | +> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 97.2% (-2.8 points) | 97.2% → 91.7% (-5.5 points) | +| Correctness | 64.4% → 96.7% (+32.3 points) | 77.8% → 91.1% (+13.3 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 37.1% → 93.8% (+56.7 points) | 58.8% → 94.2% (+35.4 points) | +| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 64.7% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 96.2% (-2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 89.3% → 88.2% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | ## Skill Version(s):
0.3.0 (source: changelog, released 2025-05-21)
diff --git a/skills/nvidia-skill-finder/skill.oms.sig b/skills/nvidia-skill-finder/skill.oms.sig index cdde023c3..100ab5f98 100644 --- a/skills/nvidia-skill-finder/skill.oms.sig +++ b/skills/nvidia-skill-finder/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIwNTM4NDcwMjkyNDZlYjI1NTA2YzMyZjk0MDQ5NDJlOGMyNzM2MzdiYjA1ODI5MzhlMjAwYTU5ZGQ5Mzg0YmE1IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiZWJjN2RlMTc5OGIyYmJhZTE1ZjExOGMyNWVjZjQwOThkYThlN2Q4MmVhYjJhMjhjZWEyNTgwNTA1ZjQwZTBjOSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJmNTllNWMyYzllODljZDM1NjM3NjQyMjUyODBmOWE4ZjA1ZWExMTY0ZWZhOTc4OGZlYTU4OTJmNjViMDRkZjJjIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFnZW50cy9vcGVuYWkueWFtbCIsCiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiNzE2ZmNiM2ZiMTFkMDM2MjYyYTY5NjY0OTI4MmZlYTc2ZmE2MGQyNDIzYTc4MDFlMWQ0YTNlOGY3OTMzZjBlNCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3RheG9ub215LXJvdXRpbmcubWQiLAogICAgICAgICJkaWdlc3QiOiAiYTI0YjU5NGIwNmZhNGUxNmM4OTM3ZGE1NGI5YmZjMmNiZWNlOGJjYzY2YjlkMDJhNjcyOTQzNDAyMWExNjg0OCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjI5OWRkZjA3NzUzOTU3ODY2MzQxMzQ5YjQ5N2MxODQ4MWNmYzI1NWE5MTMxZWYzNTAyNDM4Y2Y3NDA3MTg3OTAiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMGGg1+VbtPZ6EXlNDc0xHBMjfZN0EAGw1oDeoYkA/BuRWHpci0EFllec/wl0PyaTswIxAJLsjn6E8DOrgQvHZgdt+8ILmYrhydTiQ6wLzhXUeJKe3vGsgSFDv0BLA1hZ1kp1ig==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJkMzNkNGU2NWE1NDAwN2EzMDJkNDg2M2IxNDgzYzkzMDA4YTBmOTRkZDAzNGIzMTZkZGIwZTE5MmU1ZjZjMGQ3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYzVmMjM3OWUxZTdhMTAyNjhlNzEzMGMzNTk2N2Y1OTBjMTkxNzdiZDQ5OGQ4ZWJjZTY5NjU2YzA5ZDQwNmNlNCIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzQ0MmM1YTA3NTE2NTc3YWJhN2U5OGQ2MjkyMzE2MmI4YWY2ZmM1OTNkNmZmYmFlMWRmMzFjZDU2Njk3ZjRmZSIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IiwKICAgICAgICAibmFtZSI6ICJhZ2VudHMvb3BlbmFpLnlhbWwiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzZTQwYjc0ZjM3MWZkMzAzMDIzODI3MDE0MWE2N2FiMmM2OGEzNTJjODNmYzcxNmRkOTQ0MjcwODA3M2M0NzVhIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMTFjNGNhOWUxMWZjMzZjMDQwYjFlMjgwZGRkYmNiZTg5ZGI0ZmExMTIwOGI2Y2UzNjU4YjJlYzkyNjE2N2U0ZiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9oZXJtZXMtY2F0YWxvZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImEyNGI1OTRiMDZmYTRlMTZjODkzN2RhNTRiOWJmYzJjYmVjZThiY2M2NmI5ZDAyYTY3Mjk0MzQwMjFhMTY4NDgiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdGF4b25vbXktcm91dGluZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjBhMDkzN2Y3YjA3ZTViMWVkYzhjMjdiNzBmZjg2NWM2OTg3NzZkNGFmNTU4ZTcxYjQxMGFkYTlhZDJiMDI1ZWUiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDsS+e34JG8dwuEyIgealbx6MuWJKxHK1JCOy4R0w9zy4pvixnIHDB7ATbKlkFX3XUCMD3DtNLaf4gBFLqsCfmamscklrLP1nc6uYTQhVvgPMpiPoFLPsg6OBtJXTE2uRtuYQ==","keyid":""}]}} \ No newline at end of file From d75522accb86815ff57a53c4ed646bb212c36b00 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Thu, 5 Nov 2026 10:51:14 -0500 Subject: [PATCH 08/19] fix(nvidia-skill-finder): raise the Hermes search limit past the default 25 hermes skills search returns 25 results by default, which silently truncates the broad single-keyword queries this guidance asks for: jetson matches 39 skills, so jetson-quick-start was cut off and never recommended. Pass --limit 100 on every search. Also note that --source nvidia requires Hermes 0.22 or newer, and give the fallback for older installs. Reported by teknium1 on NVIDIA/skills#820. Signed-off-by: Jason Dudash --- plugins/nvidia-skills/README.md | 2 +- .../nvidia-skill-finder/references/hermes-catalog.md | 10 +++++++++- .../nvidia-skill-finder/references/hermes-catalog.md | 10 +++++++++- versions.json | 4 ++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/plugins/nvidia-skills/README.md b/plugins/nvidia-skills/README.md index eb9385345..e1d507732 100644 --- a/plugins/nvidia-skills/README.md +++ b/plugins/nvidia-skills/README.md @@ -18,7 +18,7 @@ You can also get it by adding the self-hosted `nvidia-official` marketplace from Hermes ships its own scanned, profile-aware skills catalog, so the finder uses that instead and never calls `npx`. The details live in the skill's `references/hermes-catalog.md`. -- **Catalog lookup.** `hermes skills search --source nvidia --json`. That search matches the query as a plain substring, so the finder queries one broad keyword at a time; if several come back empty it falls back to the full catalog listing on `raw.githubusercontent.com`, so a narrow search term can't hide a skill that exists. +- **Catalog lookup.** `hermes skills search --source nvidia --json --limit 100`. That search matches the query as a plain substring, so the finder queries one broad keyword at a time; `--limit 100` raises the default of 25 so broad keywords aren't truncated. If several keywords come back empty it falls back to the full catalog listing on `raw.githubusercontent.com`, so a narrow search term can't hide a skill that exists. Requires Hermes 0.22+ for `--source`. - **Installs only with approval.** `hermes skills install NVIDIA/skills/`, which runs Hermes's own security scan and installs into the active profile. Hermes prompts for confirmation itself, so the finder does not pass `--yes`. The plugin itself contains no code, stores no credentials, and sends no telemetry. The `skills` CLI follows its own policies. diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md index 486206d7f..0030f2f5d 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -4,12 +4,20 @@ Follow this instead of the `npx` commands in SKILL.md when you are running in Hermes Agent. Hermes has a built-in catalog that is already security scanned and profile-aware, so `npx` is both unnecessary and far more expensive in context. +`--source nvidia` requires Hermes 0.22 or newer; older versions reject the +flag. If it is rejected, drop `--source nvidia` and filter the results for +`NVIDIA/skills` identifiers yourself. + ## Catalog lookup ```bash -hermes skills search --source nvidia --json +hermes skills search --source nvidia --json --limit 100 ``` +Always pass `--limit 100`. The default is 25, which silently truncates the +broad keywords this guidance tells you to use — `jetson` alone matches 39 +skills, so the default cuts off real results such as `jetson-quick-start`. + Search one broad keyword at a time — `jetson`, `usd`, `dynamo`, `pandas`, `cuda`. `--json` returns each candidate's full description; read the descriptions and choose the match yourself. The search ranks lexically, so it diff --git a/skills/nvidia-skill-finder/references/hermes-catalog.md b/skills/nvidia-skill-finder/references/hermes-catalog.md index 486206d7f..0030f2f5d 100644 --- a/skills/nvidia-skill-finder/references/hermes-catalog.md +++ b/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -4,12 +4,20 @@ Follow this instead of the `npx` commands in SKILL.md when you are running in Hermes Agent. Hermes has a built-in catalog that is already security scanned and profile-aware, so `npx` is both unnecessary and far more expensive in context. +`--source nvidia` requires Hermes 0.22 or newer; older versions reject the +flag. If it is rejected, drop `--source nvidia` and filter the results for +`NVIDIA/skills` identifiers yourself. + ## Catalog lookup ```bash -hermes skills search --source nvidia --json +hermes skills search --source nvidia --json --limit 100 ``` +Always pass `--limit 100`. The default is 25, which silently truncates the +broad keywords this guidance tells you to use — `jetson` alone matches 39 +skills, so the default cuts off real results such as `jetson-quick-start`. + Search one broad keyword at a time — `jetson`, `usd`, `dynamo`, `pandas`, `cuda`. `--json` returns each candidate's full description; read the descriptions and choose the match yourself. The search ranks lexically, so it diff --git a/versions.json b/versions.json index 07481015d..aa37ba37c 100644 --- a/versions.json +++ b/versions.json @@ -2119,8 +2119,8 @@ "name": "nvidia-skill-finder", "path": "skills/nvidia-skill-finder", "content_digest": "sha256:da43951c0c8c69da5622f5299055228c70bc6345273e7869f3c59acb92af8310", - "last_commit": "dce10b59d664f5939c58602cadc4db28fab768be", - "last_modified": "2026-11-04" + "last_commit": "828a6b398c7d59665f91b174b40438cc56150a17", + "last_modified": "2026-11-05" }, { "name": "nvshmem-collect-performance-data", From a16b859e4cfbf706739a17c379629f4c0d1a9ad1 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Thu, 5 Nov 2026 10:52:15 -0500 Subject: [PATCH 09/19] chore(versions): refresh nvidia-skill-finder entry Signed-off-by: Jason Dudash --- versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.json b/versions.json index aa37ba37c..e864c0323 100644 --- a/versions.json +++ b/versions.json @@ -2119,7 +2119,7 @@ "name": "nvidia-skill-finder", "path": "skills/nvidia-skill-finder", "content_digest": "sha256:da43951c0c8c69da5622f5299055228c70bc6345273e7869f3c59acb92af8310", - "last_commit": "828a6b398c7d59665f91b174b40438cc56150a17", + "last_commit": "24cb45e1e1882853ba911fb661367fa20bde4615", "last_modified": "2026-11-05" }, { From 378b88e9f573ffc62594ae28184b3ffae1642da3 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Thu, 5 Nov 2026 10:53:11 -0500 Subject: [PATCH 10/19] chore(plugins): sync nvidia-skills with the attached signatures Signed-off-by: Jason Dudash --- .../skills/nvidia-skill-finder/BENCHMARK.md | 104 +++++++++--------- .../skills/nvidia-skill-finder/skill-card.md | 47 ++++---- .../skills/nvidia-skill-finder/skill.oms.sig | 2 +- 3 files changed, 76 insertions(+), 77 deletions(-) diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md index 0337d86f8..263e29911 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 94.1% — baseline ran, but no comparable score was available; uplift unavailable | 87.1% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 97.2% → 97.2% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 62.2% → 96.7% (+34.5 points) | 77.8% → 91.1% (+13.3 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 39.9% → 96.5% (+56.6 points) | 59.3% → 92.1% (+32.8 points) | -| Efficiency | 80.3% — baseline ran, but no comparable score was available; uplift unavailable | 61.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 97.2% (-2.8 points) | 97.2% → 91.7% (-5.5 points) | +| Correctness | 64.4% → 96.7% (+32.3 points) | 77.8% → 91.1% (+13.3 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 37.1% → 93.8% (+56.7 points) | 58.8% → 94.2% (+35.4 points) | +| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 64.7% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 96.4% (-1.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 96.2% (-2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.0% → 88.9% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.3% → 88.2% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,45 +80,45 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 2,746,750 | 1,692,842 | +1,053,908 | +62.26% | skill 18/18; base 18/18 | -| claude-code | nvidia-skill-finder-diff-bare-invocation | 158,485 | 70,559 | +87,926 | +124.61% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 160,553 | 107,298 | +53,255 | +49.63% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-deep-research | 153,825 | 35,323 | +118,502 | +335.48% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-express-route | 224,006 | 182,756 | +41,250 | +22.57% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 109,046 | 142,661 | -33,615 | -23.56% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-python-refactor | 107,662 | 106,575 | +1,087 | +1.02% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-react-optimize | 144,923 | 107,108 | +37,815 | +35.31% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-video-editing | 108,246 | 71,223 | +37,023 | +51.98% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-cad-to-simready | 153,252 | 143,560 | +9,692 | +6.75% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dicom | 154,197 | 35,935 | +118,262 | +329.10% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dynamo-kv | 195,216 | 144,272 | +50,944 | +35.31% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,152 | 107,398 | +45,754 | +42.60% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,942 | 36,723 | +118,219 | +321.92% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,508 | 35,926 | +117,582 | +327.29% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,903 | 107,200 | +46,703 | +43.57% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,782 | 151,902 | +2,880 | +1.90% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,925 | 70,956 | +82,969 | +116.93% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,127 | 35,467 | +117,660 | +331.75% | skill 1/1; base 1/1 | -| codex | All cases | 1,130,804 | 610,948 | +519,856 | +85.09% | skill 18/18; base 18/18 | -| codex | nvidia-skill-finder-diff-bare-invocation | 68,890 | 35,919 | +32,971 | +91.79% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-browse-no-keywords | 89,599 | 52,784 | +36,815 | +69.75% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-deep-research | 90,172 | 35,866 | +54,306 | +151.41% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-express-route | 55,483 | 40,948 | +14,535 | +35.50% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,915 | 27,157 | +14,758 | +54.34% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-python-refactor | 41,302 | 40,437 | +865 | +2.14% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-react-optimize | 40,956 | 41,097 | -141 | -0.34% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-video-editing | 13,505 | 13,316 | +189 | +1.42% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-cad-to-simready | 67,881 | 13,365 | +54,516 | +407.90% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dicom | 73,056 | 38,531 | +34,525 | +89.60% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dynamo-kv | 17,983 | 31,520 | -13,537 | -42.95% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-gpu-pandas | 69,058 | 22,021 | +47,037 | +213.60% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-isaac-lab | 106,097 | 56,158 | +49,939 | +88.93% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-driver-install | 69,157 | 17,804 | +51,353 | +288.44% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-setup | 69,085 | 17,655 | +51,430 | +291.31% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-multigpu-training | 77,888 | 22,466 | +55,422 | +246.69% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-openusd-optimization | 68,381 | 65,091 | +3,290 | +5.05% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-vehicle-routing | 70,396 | 38,813 | +31,583 | +81.37% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 3,877,554 | 2,303,790 | +1,573,764 | +68.31% | skill 36/36; base 36/36 | +| claude-code | All cases | 2,940,059 | 1,733,228 | +1,206,831 | +69.63% | skill 18/18; base 18/18 | +| claude-code | nvidia-skill-finder-diff-bare-invocation | 159,556 | 107,051 | +52,505 | +49.05% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 160,640 | 106,788 | +53,852 | +50.43% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-deep-research | 349,223 | 35,273 | +313,950 | +890.06% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-express-route | 185,632 | 223,103 | -37,471 | -16.80% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 72,345 | 71,645 | +700 | +0.98% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-python-refactor | 142,972 | 141,956 | +1,016 | +0.72% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-react-optimize | 145,032 | 142,851 | +2,181 | +1.53% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-video-editing | 107,845 | 71,165 | +36,680 | +51.54% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-cad-to-simready | 154,290 | 143,642 | +10,648 | +7.41% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dicom | 154,191 | 35,577 | +118,614 | +333.40% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dynamo-kv | 231,606 | 144,237 | +87,369 | +60.57% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,338 | 108,293 | +45,045 | +41.60% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,754 | 72,321 | +82,433 | +113.98% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,792 | 35,680 | +118,112 | +331.03% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,387 | 71,488 | +81,899 | +114.56% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,938 | 151,301 | +3,637 | +2.40% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,414 | 35,400 | +118,014 | +333.37% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,104 | 35,457 | +117,647 | +331.80% | skill 1/1; base 1/1 | +| codex | All cases | 1,208,315 | 806,320 | +401,995 | +49.86% | skill 18/18; base 18/18 | +| codex | nvidia-skill-finder-diff-bare-invocation | 86,372 | 35,913 | +50,459 | +140.50% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-browse-no-keywords | 93,587 | 47,277 | +46,310 | +97.95% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-deep-research | 80,712 | 187,624 | -106,912 | -56.98% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-express-route | 40,930 | 40,522 | +408 | +1.01% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,679 | 27,181 | +14,498 | +53.34% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-python-refactor | 40,995 | 40,331 | +664 | +1.65% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-react-optimize | 69,618 | 40,882 | +28,736 | +70.29% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-video-editing | 13,444 | 13,257 | +187 | +1.41% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-cad-to-simready | 67,899 | 13,327 | +54,572 | +409.48% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dicom | 126,618 | 57,984 | +68,634 | +118.37% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dynamo-kv | 71,047 | 17,627 | +53,420 | +303.06% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-gpu-pandas | 53,320 | 26,172 | +27,148 | +103.73% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-isaac-lab | 103,549 | 52,434 | +51,115 | +97.48% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-driver-install | 61,973 | 17,645 | +44,328 | +251.22% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-setup | 53,437 | 17,828 | +35,609 | +199.74% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-multigpu-training | 78,115 | 66,339 | +11,776 | +17.75% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-openusd-optimization | 54,639 | 65,106 | -10,467 | -16.08% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-vehicle-routing | 70,381 | 38,871 | +31,510 | +81.06% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 4,148,374 | 2,539,548 | +1,608,826 | +63.35% | skill 36/36; base 36/36 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -126,7 +126,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | @@ -135,12 +135,12 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c
Show detailed findings and successful checks -- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nvidia-skill-finder/SKILL.md`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:74`) -- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:135`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:103`) -- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:148`) -- 9 additional finding(s) are available in the full evaluation artifacts. +- **MEDIUM** SECURITY/Security finding (SQP-2): Because the skill triggers on many NVIDIA-related requests without being asked, the agent may download and run remote np (`SKILL.md:109`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:113`) +- 11 additional finding(s) are available in the full evaluation artifacts.
diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md index cbaebadd5..4e8547b49 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill.
+Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill; it checks the live NVIDIA skills catalog and recommends matching skills, while staying quiet for generic non-NVIDIA route, optimize, deploy, AI, video, data, or infrastructure tasks.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
CC-BY-4.0 AND Apache-2.0
## Use Case:
-Developers and engineers working with NVIDIA products, hardware, SDKs, and services use this skill to discover, recommend, and (with explicit approval) install the most relevant NVIDIA agent skill from the live NVIDIA skills catalog for their task.
+Developers and engineers working with NVIDIA products, hardware, SDKs, and services use this skill to discover, evaluate, and (with explicit approval) install relevant NVIDIA agent skills from the live catalog, including Hermes Agent catalog lookups.
### Deployment Geography for Use:
Global
@@ -25,19 +25,18 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [references/taxonomy-routing.md](references/taxonomy-routing.md)
-- [references/hermes-catalog.md](references/hermes-catalog.md)
-- [NVIDIA Skills on GitHub](https://github.com/NVIDIA/skills/tree/main/skills)
+- [Hermes Agent: Catalog Lookup and Install](references/hermes-catalog.md)
+- [NVIDIA Taxonomy Routing](references/taxonomy-routing.md)
+- [NVIDIA Skills catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
- [NVIDIA Skills on build.nvidia.com](https://build.nvidia.com/skills)
- [NVIDIA Skills catalog index (skills.sh.json)](https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json)
-- [NVIDIA Skills documentation](https://docs.nvidia.com/skills)
## Skill Output:
**Output Type(s):** [Analysis, Shell commands]
**Output Format:** [Markdown with inline bash code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Recommends at most three catalog skills ordered by confidence; install commands are proposed only and run only after explicit user approval.]
+**Other Properties Related to Output:** [Recommends at most three skills ordered by confidence; installs only after explicit user approval.]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -46,7 +45,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-18 evaluation tasks (13 positive, 5 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox`), compared against a no-skill baseline.
+18 evaluation tasks (13 positive, 5 negative), 1 attempt per task, run in the `k8s-sandbox` environment with each task attempt in its own isolated sandbox pod; each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -57,13 +56,13 @@ Reported benchmark dimensions:
- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Final-answer correctness against the reference answer.
-- `goal_accuracy`: Whether the user's goal was achieved.
-- `behavior_check`: Whether the expected workflow behavior was followed.
-- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
@@ -78,12 +77,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 94.1% — baseline ran, but no comparable score was available; uplift unavailable | 87.1% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 97.2% → 97.2% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 62.2% → 96.7% (+34.5 points) | 77.8% → 91.1% (+13.3 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 39.9% → 96.5% (+56.6 points) | 59.3% → 92.1% (+32.8 points) | -| Efficiency | 80.3% — baseline ran, but no comparable score was available; uplift unavailable | 61.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 97.2% (-2.8 points) | 97.2% → 91.7% (-5.5 points) | +| Correctness | 64.4% → 96.7% (+32.3 points) | 77.8% → 91.1% (+13.3 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 37.1% → 93.8% (+56.7 points) | 58.8% → 94.2% (+35.4 points) | +| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 64.7% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -98,15 +97,15 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 96.4% (-1.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 96.2% (-2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.0% → 88.9% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.3% → 88.2% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ### Tier Status | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig index b4d8ad419..100ab5f98 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIyNjQ3MzJjOGVmZDk5MDJkMTI5MjA3NTlkZTkwMjg0YWYyYmYxNDdlZGNkNjg1YTIyY2IzYTk4ZTcyODhjNzY0IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjVkODcyMDgwZDg0YmYxZDU2OWNlOGM3MjFjZjRlZGM4ZWI2YTk1ZjBhMjIyYTUwOWQwMzExYWI4NDdmYjgxMmEiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjc0NDJjNWEwNzUxNjU3N2FiYTdlOThkNjI5MjMxNjJiOGFmNmZjNTkzZDZmZmJhZTFkZjMxY2Q1NjY5N2Y0ZmUiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZjJiMjk0MjI1MjA2MTI1MDU2NDNjNmExZGQyZjQ0MWI0YTFiZjM1MGZhNTFjNmZlNmVmNDVhNTI2YTcyYjczOSIsCiAgICAgICAgIm5hbWUiOiAiYWdlbnRzL29wZW5haS55YW1sIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiM2U0MGI3NGYzNzFmZDMwMzAyMzgyNzAxNDFhNjdhYjJjNjhhMzUyYzgzZmM3MTZkZDk0NDI3MDgwNzNjNDc1YSIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjExYzRjYTllMTFmYzM2YzA0MGIxZTI4MGRkZGJjYmU4OWRiNGZhMTEyMDhiNmNlMzY1OGIyZWM5MjYxNjdlNGYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvaGVybWVzLWNhdGFsb2cubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJhMjRiNTk0YjA2ZmE0ZTE2Yzg5MzdkYTU0YjliZmMyY2JlY2U4YmNjNjZiOWQwMmE2NzI5NDM0MDIxYTE2ODQ4IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3RheG9ub215LXJvdXRpbmcubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIyNzc1MjVlZTExNzQ5ZTAzZGZjYjc1MDVmZDI0Y2ExZjJiYjQ1NzIyNDZmODU1M2FiMWUxMWZhZmZmZGFjNjk0IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMGgUGS6iicxQ8OlZHs2w00O27KHwotieqogR2IHsISSLIxw9dQ5WL64Af7nez+UFdAIxAK6S4T1WMwigGm4XHTlZpKIbFCO5Qo8ISCV2SLwsrEXHcvKRtdRX3KZHCFQY2dxytQ==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJkMzNkNGU2NWE1NDAwN2EzMDJkNDg2M2IxNDgzYzkzMDA4YTBmOTRkZDAzNGIzMTZkZGIwZTE5MmU1ZjZjMGQ3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYzVmMjM3OWUxZTdhMTAyNjhlNzEzMGMzNTk2N2Y1OTBjMTkxNzdiZDQ5OGQ4ZWJjZTY5NjU2YzA5ZDQwNmNlNCIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzQ0MmM1YTA3NTE2NTc3YWJhN2U5OGQ2MjkyMzE2MmI4YWY2ZmM1OTNkNmZmYmFlMWRmMzFjZDU2Njk3ZjRmZSIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IiwKICAgICAgICAibmFtZSI6ICJhZ2VudHMvb3BlbmFpLnlhbWwiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzZTQwYjc0ZjM3MWZkMzAzMDIzODI3MDE0MWE2N2FiMmM2OGEzNTJjODNmYzcxNmRkOTQ0MjcwODA3M2M0NzVhIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMTFjNGNhOWUxMWZjMzZjMDQwYjFlMjgwZGRkYmNiZTg5ZGI0ZmExMTIwOGI2Y2UzNjU4YjJlYzkyNjE2N2U0ZiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9oZXJtZXMtY2F0YWxvZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImEyNGI1OTRiMDZmYTRlMTZjODkzN2RhNTRiOWJmYzJjYmVjZThiY2M2NmI5ZDAyYTY3Mjk0MzQwMjFhMTY4NDgiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdGF4b25vbXktcm91dGluZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjBhMDkzN2Y3YjA3ZTViMWVkYzhjMjdiNzBmZjg2NWM2OTg3NzZkNGFmNTU4ZTcxYjQxMGFkYTlhZDJiMDI1ZWUiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDsS+e34JG8dwuEyIgealbx6MuWJKxHK1JCOy4R0w9zy4pvixnIHDB7ATbKlkFX3XUCMD3DtNLaf4gBFLqsCfmamscklrLP1nc6uYTQhVvgPMpiPoFLPsg6OBtJXTE2uRtuYQ==","keyid":""}]}} \ No newline at end of file From 6dc671267aa8ddbd423d04df5200a3e1914d0595 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Thu, 5 Nov 2026 10:53:43 -0500 Subject: [PATCH 11/19] chore(versions): refresh after signature attach Signed-off-by: Jason Dudash --- versions.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions.json b/versions.json index e864c0323..99a6fe644 100644 --- a/versions.json +++ b/versions.json @@ -2118,8 +2118,8 @@ { "name": "nvidia-skill-finder", "path": "skills/nvidia-skill-finder", - "content_digest": "sha256:da43951c0c8c69da5622f5299055228c70bc6345273e7869f3c59acb92af8310", - "last_commit": "24cb45e1e1882853ba911fb661367fa20bde4615", + "content_digest": "sha256:b5c517e02618ea17df54fe204435adff5bcbd65716d56f4f8a5ec97eced30a0d", + "last_commit": "7fb950afa64022b230b843efb3558b94eea98787", "last_modified": "2026-11-05" }, { From f658f979306be1bcd4b19f89ab4c67e24956173f Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Thu, 5 Nov 2026 10:56:57 -0500 Subject: [PATCH 12/19] chore(benchmarks): refresh after signature attach Signed-off-by: Jason Dudash --- benchmarks.json | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/benchmarks.json b/benchmarks.json index bed118c15..994b43657 100644 --- a/benchmarks.json +++ b/benchmarks.json @@ -6025,13 +6025,13 @@ }, { "skill": "nvidia-skill-finder", - "evaluation_date": "2026-10-01", + "evaluation_date": "2026-11-04", "environment": "k8s-sandbox", - "evaluator_version": "1.5.6", - "dataset_digest": "sha256:df412e229dfed3687e1112c8f5bc439a6f3695291f3d357f049002621e90845f", - "validation_status": null, + "evaluator_version": "1.6.6", + "dataset_digest": "sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281", + "validation_status": "passed", "tasks": 18, - "attempts_per_task": 3, + "attempts_per_task": 1, "pass_threshold_pct": null, "verdict": "PASS", "agents": [ @@ -6041,7 +6041,7 @@ "catalog_dir": "nvidia-skill-finder", "component": "NVIDIA Skills", "has_results": true, - "average_uplift_pct": 26.97 + "average_uplift_pct": 21.57 }, { "skill": "nvshmem-collect-performance-data", @@ -34187,7 +34187,7 @@ "num": null, "agent": "claude-code", "score_pct": 97.2, - "uplift_pct": -1.0 + "uplift_pct": -2.8 }, { "catalog_dir": "nvidia-skill-finder", @@ -34195,8 +34195,8 @@ "dimension": "Security", "num": null, "agent": "codex", - "score_pct": 94.4, - "uplift_pct": 5.9 + "score_pct": 91.7, + "uplift_pct": -5.5 }, { "catalog_dir": "nvidia-skill-finder", @@ -34205,7 +34205,7 @@ "num": null, "agent": "claude-code", "score_pct": 96.7, - "uplift_pct": 27.1 + "uplift_pct": 32.3 }, { "catalog_dir": "nvidia-skill-finder", @@ -34213,8 +34213,8 @@ "dimension": "Correctness", "num": null, "agent": "codex", - "score_pct": 100.0, - "uplift_pct": 29.2 + "score_pct": 91.1, + "uplift_pct": 13.3 }, { "catalog_dir": "nvidia-skill-finder", @@ -34231,7 +34231,7 @@ "dimension": "Discoverability", "num": null, "agent": "codex", - "score_pct": 94.6, + "score_pct": 95.0, "uplift_pct": null }, { @@ -34240,8 +34240,8 @@ "dimension": "Effectiveness", "num": null, "agent": "claude-code", - "score_pct": 87.9, - "uplift_pct": 51.8 + "score_pct": 93.8, + "uplift_pct": 56.7 }, { "catalog_dir": "nvidia-skill-finder", @@ -34249,8 +34249,8 @@ "dimension": "Effectiveness", "num": null, "agent": "codex", - "score_pct": 89.7, - "uplift_pct": 48.8 + "score_pct": 94.2, + "uplift_pct": 35.4 }, { "catalog_dir": "nvidia-skill-finder", @@ -34258,7 +34258,7 @@ "dimension": "Efficiency", "num": null, "agent": "claude-code", - "score_pct": 78.4, + "score_pct": 79.6, "uplift_pct": null }, { @@ -34267,7 +34267,7 @@ "dimension": "Efficiency", "num": null, "agent": "codex", - "score_pct": 66.5, + "score_pct": 64.7, "uplift_pct": null }, { From 6ae4b5e52e84a5d446bb278023700cedc880dd23 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Thu, 5 Nov 2026 16:19:19 -0500 Subject: [PATCH 13/19] feat(nvidia-skill-finder): pull the full catalog in Hermes instead of searching Hermes search matches the query as a plain case-insensitive substring over each entry's joined fields, so it misses phrases and truncates broad keywords at the default limit of 25. Working around that took ~8 searches per question, which cost more context than reading the whole catalog and could still miss the right skill. Read the 437-entry catalog listing once and choose from it, then confirm the identifier with a single narrow search before installing through Hermes, which keeps the security scan and profile awareness. Across the 22-question product sweep this cut searches from 176 to 21 and transcripts from 250KB to 46KB at equal accuracy. Signed-off-by: Jason Dudash --- plugins/nvidia-skills/README.md | 4 +- .../skills/nvidia-skill-finder/SKILL.md | 4 +- .../references/hermes-catalog.md | 83 +++++++------------ skills/nvidia-skill-finder/SKILL.md | 4 +- .../references/hermes-catalog.md | 83 +++++++------------ 5 files changed, 70 insertions(+), 108 deletions(-) diff --git a/plugins/nvidia-skills/README.md b/plugins/nvidia-skills/README.md index e1d507732..f4dea0b84 100644 --- a/plugins/nvidia-skills/README.md +++ b/plugins/nvidia-skills/README.md @@ -18,8 +18,8 @@ You can also get it by adding the self-hosted `nvidia-official` marketplace from Hermes ships its own scanned, profile-aware skills catalog, so the finder uses that instead and never calls `npx`. The details live in the skill's `references/hermes-catalog.md`. -- **Catalog lookup.** `hermes skills search --source nvidia --json --limit 100`. That search matches the query as a plain substring, so the finder queries one broad keyword at a time; `--limit 100` raises the default of 25 so broad keywords aren't truncated. If several keywords come back empty it falls back to the full catalog listing on `raw.githubusercontent.com`, so a narrow search term can't hide a skill that exists. Requires Hermes 0.22+ for `--source`. -- **Installs only with approval.** `hermes skills install NVIDIA/skills/`, which runs Hermes's own security scan and installs into the active profile. Hermes prompts for confirmation itself, so the finder does not pass `--yes`. +- **Catalog lookup.** The finder reads the full catalog listing from `raw.githubusercontent.com` and picks from it, rather than searching. Hermes search matches the query as a plain substring, so it misses phrases and truncates broad keywords at a default of 25; reading the whole list once is both cheaper in context and complete. +- **Installs only with approval.** Once a skill is chosen, the finder confirms its identifier with `hermes skills search --source nvidia --json --limit 100` (requires Hermes 0.22+ for `--source`), then installs with `hermes skills install `, which runs Hermes's own security scan and installs into the active profile. Hermes prompts for confirmation itself, so the finder does not pass `--yes`. The plugin itself contains no code, stores no credentials, and sends no telemetry. The `skills` CLI follows its own policies. diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md index 4bea2e239..3083c8e59 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md @@ -113,8 +113,8 @@ Catalog check means one of: npx skills add nvidia/skills --list ``` -- If running in Hermes Agent, use the Hermes CLI instead of `npx` and follow - [references/hermes-catalog.md](references/hermes-catalog.md). +- If running in Hermes Agent, do not run `npx`. Follow + [references/hermes-catalog.md](references/hermes-catalog.md) instead. - Otherwise do not run `hermes` commands. The binary may exist on the machine even when you are a different agent, and installing through it writes to Hermes's skills directory, where you cannot load the result. diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md index 0030f2f5d..957e572bb 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -1,8 +1,8 @@ # Hermes Agent: Catalog Lookup and Install Follow this instead of the `npx` commands in SKILL.md when you are running in -Hermes Agent. Hermes has a built-in catalog that is already security scanned and -profile-aware, so `npx` is both unnecessary and far more expensive in context. +Hermes Agent. Install goes through Hermes so the skill is security scanned and +lands in the active profile, and `npx` stays unnecessary. `--source nvidia` requires Hermes 0.22 or newer; older versions reject the flag. If it is rejected, drop `--source nvidia` and filter the results for @@ -10,73 +10,54 @@ flag. If it is rejected, drop `--source nvidia` and filter the results for ## Catalog lookup +List the whole catalog once and choose from it yourself: + ```bash -hermes skills search --source nvidia --json --limit 100 +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ + | jq -r '.skills[] | "\(.name): \(.description[0:110])"' ``` -Always pass `--limit 100`. The default is 25, which silently truncates the -broad keywords this guidance tells you to use — `jetson` alone matches 39 -skills, so the default cuts off real results such as `jetson-quick-start`. - -Search one broad keyword at a time — `jetson`, `usd`, `dynamo`, `pandas`, -`cuda`. `--json` returns each candidate's full description; read the -descriptions and choose the match yourself. The search ranks lexically, so it -narrows the catalog but does not do the matching for you. Do not assume the -first row is the answer. - -## Why single keywords, not phrases - -Hermes matches a query as a plain case-insensitive substring against each -entry's name, description, tags, identifier, and provider joined together. It -does not tokenize, so a multi-word query only matches when those exact words -appear consecutively in that order. +If `jq` is unavailable, pipe the same URL through +`python3 -c 'import json,sys; [print(f"{s[\"name\"]}: {s[\"description\"][:110]}") for s in json.load(sys.stdin)["skills"]]'`. -Descriptive phrases therefore collapse to zero results while the obvious -keyword works: +Read the list and pick the best match on the descriptions. Prefer this over +searching. Searching looks cheaper but is not: it matches the query as a plain +case-insensitive substring over each entry's joined fields, so it misses +phrases (`jetson pinmux` returns nothing even though `jetson-customize-pinmux` +exists), and broad keywords return full descriptions for every hit. A handful +of searches costs more context than the complete listing above and can still +miss the right skill. -| Query | Result | -| --- | --- | -| `jetson pinmux` | no rows, even though `jetson-customize-pinmux` exists | -| `jetson` | returns `jetson-customize-pinmux` and the rest of the family | -| `OpenUSD` | returns only this finder | -| `usd` | returns the Omniverse and USD skills | +Do not use `hermes skills browse` either. It costs more than the listing and +truncates both names and descriptions, so it is worse on accuracy and cost. -Run two or three separate single-keyword searches, widening by one step each -time: +## Confirm the identifier -1. The product name as users say it — `jetson`, `dynamo`, `cuopt`. -2. The product family — `omniverse`, `rapids`, `nemo`, `holoscan`. -3. The taxonomy lane — `usd`, `routing`, `inference`, `genomics`. - -## Full-catalog fallback - -If several single-keyword searches return nothing, list the whole catalog and -choose from it before giving up: +Once you have chosen a specific skill, look up that exact name to confirm it is +available in Hermes and to get its identifier: ```bash -curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ - | jq -r '.skills[] | "\(.name): \(.description)"' +hermes skills search --source nvidia --json --limit 100 ``` -Do not use `hermes skills browse` for this. It costs more context than the -listing above and truncates both names and descriptions, so it is the one -option that is worse on both accuracy and cost. +An exact skill name is a narrow query, so this stays small. Pass `--limit 100` +anyway; the default is 25, which silently truncates broader matches. ## Install +Use the `Identifier` from that result, copied verbatim: + ```bash -hermes skills install NVIDIA/skills/ +hermes skills install ``` -Casing is what determines trust, not the number of path segments. `NVIDIA/...` -resolves to the trusted first-party source; `nvidia/...` resolves to a -community-trust mirror with a placeholder description. Always use `NVIDIA`. +Casing determines trust, not the number of path segments. `NVIDIA/...` resolves +to the trusted first-party source; `nvidia/...` resolves to a community-trust +mirror with a placeholder description. Always use `NVIDIA`. -Prefer the `Identifier` from the search result, copied verbatim, over -assembling one by hand. It may be a longer path such as -`NVIDIA/skills/skills/`; that form and the short -`NVIDIA/skills/` both resolve to the same trusted skill with the -same scan hash. +The identifier may be a longer path such as `NVIDIA/skills/skills/`; +that form and the short `NVIDIA/skills/` both resolve to the same +trusted skill with the same scan hash. Confirm an identifier you are unsure about before installing it: diff --git a/skills/nvidia-skill-finder/SKILL.md b/skills/nvidia-skill-finder/SKILL.md index 4bea2e239..3083c8e59 100644 --- a/skills/nvidia-skill-finder/SKILL.md +++ b/skills/nvidia-skill-finder/SKILL.md @@ -113,8 +113,8 @@ Catalog check means one of: npx skills add nvidia/skills --list ``` -- If running in Hermes Agent, use the Hermes CLI instead of `npx` and follow - [references/hermes-catalog.md](references/hermes-catalog.md). +- If running in Hermes Agent, do not run `npx`. Follow + [references/hermes-catalog.md](references/hermes-catalog.md) instead. - Otherwise do not run `hermes` commands. The binary may exist on the machine even when you are a different agent, and installing through it writes to Hermes's skills directory, where you cannot load the result. diff --git a/skills/nvidia-skill-finder/references/hermes-catalog.md b/skills/nvidia-skill-finder/references/hermes-catalog.md index 0030f2f5d..957e572bb 100644 --- a/skills/nvidia-skill-finder/references/hermes-catalog.md +++ b/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -1,8 +1,8 @@ # Hermes Agent: Catalog Lookup and Install Follow this instead of the `npx` commands in SKILL.md when you are running in -Hermes Agent. Hermes has a built-in catalog that is already security scanned and -profile-aware, so `npx` is both unnecessary and far more expensive in context. +Hermes Agent. Install goes through Hermes so the skill is security scanned and +lands in the active profile, and `npx` stays unnecessary. `--source nvidia` requires Hermes 0.22 or newer; older versions reject the flag. If it is rejected, drop `--source nvidia` and filter the results for @@ -10,73 +10,54 @@ flag. If it is rejected, drop `--source nvidia` and filter the results for ## Catalog lookup +List the whole catalog once and choose from it yourself: + ```bash -hermes skills search --source nvidia --json --limit 100 +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ + | jq -r '.skills[] | "\(.name): \(.description[0:110])"' ``` -Always pass `--limit 100`. The default is 25, which silently truncates the -broad keywords this guidance tells you to use — `jetson` alone matches 39 -skills, so the default cuts off real results such as `jetson-quick-start`. - -Search one broad keyword at a time — `jetson`, `usd`, `dynamo`, `pandas`, -`cuda`. `--json` returns each candidate's full description; read the -descriptions and choose the match yourself. The search ranks lexically, so it -narrows the catalog but does not do the matching for you. Do not assume the -first row is the answer. - -## Why single keywords, not phrases - -Hermes matches a query as a plain case-insensitive substring against each -entry's name, description, tags, identifier, and provider joined together. It -does not tokenize, so a multi-word query only matches when those exact words -appear consecutively in that order. +If `jq` is unavailable, pipe the same URL through +`python3 -c 'import json,sys; [print(f"{s[\"name\"]}: {s[\"description\"][:110]}") for s in json.load(sys.stdin)["skills"]]'`. -Descriptive phrases therefore collapse to zero results while the obvious -keyword works: +Read the list and pick the best match on the descriptions. Prefer this over +searching. Searching looks cheaper but is not: it matches the query as a plain +case-insensitive substring over each entry's joined fields, so it misses +phrases (`jetson pinmux` returns nothing even though `jetson-customize-pinmux` +exists), and broad keywords return full descriptions for every hit. A handful +of searches costs more context than the complete listing above and can still +miss the right skill. -| Query | Result | -| --- | --- | -| `jetson pinmux` | no rows, even though `jetson-customize-pinmux` exists | -| `jetson` | returns `jetson-customize-pinmux` and the rest of the family | -| `OpenUSD` | returns only this finder | -| `usd` | returns the Omniverse and USD skills | +Do not use `hermes skills browse` either. It costs more than the listing and +truncates both names and descriptions, so it is worse on accuracy and cost. -Run two or three separate single-keyword searches, widening by one step each -time: +## Confirm the identifier -1. The product name as users say it — `jetson`, `dynamo`, `cuopt`. -2. The product family — `omniverse`, `rapids`, `nemo`, `holoscan`. -3. The taxonomy lane — `usd`, `routing`, `inference`, `genomics`. - -## Full-catalog fallback - -If several single-keyword searches return nothing, list the whole catalog and -choose from it before giving up: +Once you have chosen a specific skill, look up that exact name to confirm it is +available in Hermes and to get its identifier: ```bash -curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ - | jq -r '.skills[] | "\(.name): \(.description)"' +hermes skills search --source nvidia --json --limit 100 ``` -Do not use `hermes skills browse` for this. It costs more context than the -listing above and truncates both names and descriptions, so it is the one -option that is worse on both accuracy and cost. +An exact skill name is a narrow query, so this stays small. Pass `--limit 100` +anyway; the default is 25, which silently truncates broader matches. ## Install +Use the `Identifier` from that result, copied verbatim: + ```bash -hermes skills install NVIDIA/skills/ +hermes skills install ``` -Casing is what determines trust, not the number of path segments. `NVIDIA/...` -resolves to the trusted first-party source; `nvidia/...` resolves to a -community-trust mirror with a placeholder description. Always use `NVIDIA`. +Casing determines trust, not the number of path segments. `NVIDIA/...` resolves +to the trusted first-party source; `nvidia/...` resolves to a community-trust +mirror with a placeholder description. Always use `NVIDIA`. -Prefer the `Identifier` from the search result, copied verbatim, over -assembling one by hand. It may be a longer path such as -`NVIDIA/skills/skills/`; that form and the short -`NVIDIA/skills/` both resolve to the same trusted skill with the -same scan hash. +The identifier may be a longer path such as `NVIDIA/skills/skills/`; +that form and the short `NVIDIA/skills/` both resolve to the same +trusted skill with the same scan hash. Confirm an identifier you are unsure about before installing it: From f73b9296d7dd2d72d0b7c0fb1228ce42730a61be Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Thu, 5 Nov 2026 16:20:25 -0500 Subject: [PATCH 14/19] chore(versions): refresh nvidia-skill-finder entry Signed-off-by: Jason Dudash --- versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.json b/versions.json index 99a6fe644..e44d2e0aa 100644 --- a/versions.json +++ b/versions.json @@ -2119,7 +2119,7 @@ "name": "nvidia-skill-finder", "path": "skills/nvidia-skill-finder", "content_digest": "sha256:b5c517e02618ea17df54fe204435adff5bcbd65716d56f4f8a5ec97eced30a0d", - "last_commit": "7fb950afa64022b230b843efb3558b94eea98787", + "last_commit": "6ae4b5e52e84a5d446bb278023700cedc880dd23", "last_modified": "2026-11-05" }, { From 4313e4911f4e7f1eeac8ca32bf9d61153d4cba9d Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Thu, 5 Nov 2026 22:46:29 +0000 Subject: [PATCH 15/19] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- .../skills/nvidia-skill-finder/BENCHMARK.md | 114 +++++++++--------- .../skills/nvidia-skill-finder/skill-card.md | 36 +++--- .../skills/nvidia-skill-finder/skill.oms.sig | 2 +- skills/nvidia-skill-finder/BENCHMARK.md | 112 ++++++++--------- skills/nvidia-skill-finder/skill-card.md | 43 +++---- skills/nvidia-skill-finder/skill.oms.sig | 2 +- 6 files changed, 152 insertions(+), 157 deletions(-) diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md index 263e29911..d6acadca1 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md @@ -10,8 +10,8 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `nvidia-skill-finder` - Validation status: `passed` -- Evaluation date: 2026-11-04 -- Evaluator version: `1.6.6` +- Evaluation date: 2026-11-05 +- Evaluator version: `1.6.8` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 18 evaluation tasks (13 positive, 5 negative) - Dataset digest: `sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281` (skill-evaluator-dataset-snapshot/1) @@ -28,11 +28,11 @@ SkillSpector provider, model, analyzer revision, and determinism evidence retain | Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | | --- | --- | --- | --- | --- | --- | --- | -| nvidia-skill-finder | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | -| nvidia-skill-finder | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | -| nvidia-skill-finder | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | -| nvidia-skill-finder | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | -| nvidia-skill-finder | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | - Controls for `nvidia-skill-finder`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 97.2% (-2.8 points) | 97.2% → 91.7% (-5.5 points) | -| Correctness | 64.4% → 96.7% (+32.3 points) | 77.8% → 91.1% (+13.3 points) | +| Overall | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 100.0% (+2.8 points) | 100.0% → 94.4% (-5.6 points) | +| Correctness | 61.1% → 94.4% (+33.3 points) | 74.4% → 93.3% (+18.9 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 37.1% → 93.8% (+56.7 points) | 58.8% → 94.2% (+35.4 points) | -| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 64.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 36.5% → 96.5% (+60.0 points) | 56.5% → 100.0% (+43.5 points) | +| Efficiency | 79.2% — baseline ran, but no comparable score was available; uplift unavailable | 63.8% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 96.2% (-2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.4% → 93.5% (-4.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.3% → 88.2% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.3% → 88.3% (-2.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,45 +80,45 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 2,940,059 | 1,733,228 | +1,206,831 | +69.63% | skill 18/18; base 18/18 | -| claude-code | nvidia-skill-finder-diff-bare-invocation | 159,556 | 107,051 | +52,505 | +49.05% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 160,640 | 106,788 | +53,852 | +50.43% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-deep-research | 349,223 | 35,273 | +313,950 | +890.06% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-express-route | 185,632 | 223,103 | -37,471 | -16.80% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 72,345 | 71,645 | +700 | +0.98% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-python-refactor | 142,972 | 141,956 | +1,016 | +0.72% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-react-optimize | 145,032 | 142,851 | +2,181 | +1.53% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-video-editing | 107,845 | 71,165 | +36,680 | +51.54% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-cad-to-simready | 154,290 | 143,642 | +10,648 | +7.41% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dicom | 154,191 | 35,577 | +118,614 | +333.40% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dynamo-kv | 231,606 | 144,237 | +87,369 | +60.57% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,338 | 108,293 | +45,045 | +41.60% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,754 | 72,321 | +82,433 | +113.98% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,792 | 35,680 | +118,112 | +331.03% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,387 | 71,488 | +81,899 | +114.56% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,938 | 151,301 | +3,637 | +2.40% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,414 | 35,400 | +118,014 | +333.37% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,104 | 35,457 | +117,647 | +331.80% | skill 1/1; base 1/1 | -| codex | All cases | 1,208,315 | 806,320 | +401,995 | +49.86% | skill 18/18; base 18/18 | -| codex | nvidia-skill-finder-diff-bare-invocation | 86,372 | 35,913 | +50,459 | +140.50% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-browse-no-keywords | 93,587 | 47,277 | +46,310 | +97.95% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-deep-research | 80,712 | 187,624 | -106,912 | -56.98% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-express-route | 40,930 | 40,522 | +408 | +1.01% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,679 | 27,181 | +14,498 | +53.34% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-python-refactor | 40,995 | 40,331 | +664 | +1.65% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-react-optimize | 69,618 | 40,882 | +28,736 | +70.29% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-video-editing | 13,444 | 13,257 | +187 | +1.41% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-cad-to-simready | 67,899 | 13,327 | +54,572 | +409.48% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dicom | 126,618 | 57,984 | +68,634 | +118.37% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dynamo-kv | 71,047 | 17,627 | +53,420 | +303.06% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-gpu-pandas | 53,320 | 26,172 | +27,148 | +103.73% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-isaac-lab | 103,549 | 52,434 | +51,115 | +97.48% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-driver-install | 61,973 | 17,645 | +44,328 | +251.22% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-setup | 53,437 | 17,828 | +35,609 | +199.74% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-multigpu-training | 78,115 | 66,339 | +11,776 | +17.75% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-openusd-optimization | 54,639 | 65,106 | -10,467 | -16.08% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-vehicle-routing | 70,381 | 38,871 | +31,510 | +81.06% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 4,148,374 | 2,539,548 | +1,608,826 | +63.35% | skill 36/36; base 36/36 | +| claude-code | All cases | 2,563,597 | 1,758,243 | +805,354 | +45.80% | skill 18/18; base 18/18 | +| claude-code | nvidia-skill-finder-diff-bare-invocation | 163,746 | 143,042 | +20,704 | +14.47% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 161,047 | 141,786 | +19,261 | +13.58% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-deep-research | 153,874 | 35,301 | +118,573 | +335.89% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-express-route | 71,576 | 220,779 | -149,203 | -67.58% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 107,929 | 107,725 | +204 | +0.19% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-python-refactor | 70,858 | 106,486 | -35,628 | -33.46% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-react-optimize | 108,001 | 106,758 | +1,243 | +1.16% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-video-editing | 107,860 | 71,136 | +36,724 | +51.63% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-cad-to-simready | 153,429 | 107,060 | +46,369 | +43.31% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dicom | 153,857 | 35,665 | +118,192 | +331.39% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dynamo-kv | 233,877 | 144,043 | +89,834 | +62.37% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,226 | 107,361 | +45,865 | +42.72% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,854 | 72,468 | +82,386 | +113.69% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,460 | 35,709 | +117,751 | +329.75% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,472 | 107,158 | +46,314 | +43.22% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-multigpu-training | 155,620 | 109,524 | +46,096 | +42.09% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,832 | 70,893 | +82,939 | +116.99% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,079 | 35,349 | +117,730 | +333.05% | skill 1/1; base 1/1 | +| codex | All cases | 1,203,358 | 676,098 | +527,260 | +77.99% | skill 18/18; base 18/18 | +| codex | nvidia-skill-finder-diff-bare-invocation | 68,816 | 39,893 | +28,923 | +72.50% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-browse-no-keywords | 52,454 | 78,017 | -25,563 | -32.77% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-deep-research | 86,657 | 36,197 | +50,460 | +139.40% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-express-route | 41,182 | 54,920 | -13,738 | -25.01% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,660 | 27,124 | +14,536 | +53.59% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-python-refactor | 41,277 | 40,337 | +940 | +2.33% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-react-optimize | 41,082 | 68,215 | -27,133 | -39.78% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-video-editing | 13,432 | 13,353 | +79 | +0.59% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-cad-to-simready | 67,885 | 13,304 | +54,581 | +410.26% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dicom | 74,804 | 30,361 | +44,443 | +146.38% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dynamo-kv | 71,043 | 41,039 | +30,004 | +73.11% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-gpu-pandas | 69,037 | 22,219 | +46,818 | +210.71% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-isaac-lab | 156,536 | 52,466 | +104,070 | +198.36% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-driver-install | 73,541 | 17,722 | +55,819 | +314.97% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-setup | 53,484 | 17,795 | +35,689 | +200.56% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-multigpu-training | 73,674 | 22,164 | +51,510 | +232.40% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-openusd-optimization | 68,956 | 65,081 | +3,875 | +5.95% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-vehicle-routing | 107,838 | 35,891 | +71,947 | +200.46% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 3,766,955 | 2,434,341 | +1,332,614 | +54.74% | skill 36/36; base 36/36 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -126,7 +126,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | @@ -135,12 +135,12 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c
Show detailed findings and successful checks -- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nvidia-skill-finder/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:74`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:103`) -- **MEDIUM** SECURITY/Security finding (SQP-2): Because the skill triggers on many NVIDIA-related requests without being asked, the agent may download and run remote np (`SKILL.md:109`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:113`) -- 11 additional finding(s) are available in the full evaluation artifacts. +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:135`) +- 9 additional finding(s) are available in the full evaluation artifacts.
diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md index 4e8547b49..6e9f9d813 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill; it checks the live NVIDIA skills catalog and recommends matching skills, while staying quiet for generic non-NVIDIA route, optimize, deploy, AI, video, data, or infrastructure tasks.
+Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill; it checks the live NVIDIA skills catalog and recommends matching NVIDIA skills (installing only with user approval), while staying quiet for generic non-NVIDIA route, optimize, deploy, AI, video, data, or infrastructure tasks.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
CC-BY-4.0 AND Apache-2.0
## Use Case:
-Developers and engineers working with NVIDIA products, hardware, SDKs, and services use this skill to discover, evaluate, and (with explicit approval) install relevant NVIDIA agent skills from the live catalog, including Hermes Agent catalog lookups.
+Developers and engineers using AI coding agents (such as Claude Code, Codex, or Hermes Agent) use this skill to discover, recommend, and—with explicit approval—install NVIDIA agent skills relevant to NVIDIA products, SDKs, hardware, and workflows such as CUDA, NIM, NeMo, Omniverse/OpenUSD, Isaac, Jetson, RAPIDS, cuOpt, Dynamo, Holoscan, TensorRT, DeepStream, and VSS.
### Deployment Geography for Use:
Global
@@ -30,11 +30,13 @@ Mitigation: Review and scan skill before deployment.
- [NVIDIA Skills catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
- [NVIDIA Skills on build.nvidia.com](https://build.nvidia.com/skills)
- [NVIDIA Skills catalog index (skills.sh.json)](https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json)
+- [NVIDIA Skills marketplace metadata (metadata.json)](https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json)
+- [NVIDIA Skills documentation](https://docs.nvidia.com/skills)
## Skill Output:
-**Output Type(s):** [Analysis, Shell commands]
-**Output Format:** [Markdown with inline bash code blocks]
+**Output Type(s):** [Analysis, Shell commands, Configuration instructions]
+**Output Format:** [Markdown with inline bash code blocks (skill recommendations with name, rationale, install command, and first prompt)]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [Recommends at most three skills ordered by confidence; installs only after explicit user approval.]
@@ -45,7 +47,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-18 evaluation tasks (13 positive, 5 negative), 1 attempt per task, run in the `k8s-sandbox` environment with each task attempt in its own isolated sandbox pod; each task was also attempted without the skill as a baseline.
+18 evaluation tasks (13 positive, 5 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment), with each task also run as a no-skill baseline. Evaluator version `1.6.8`; evaluation date 2026-11-05.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -77,12 +79,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 97.2% (-2.8 points) | 97.2% → 91.7% (-5.5 points) | -| Correctness | 64.4% → 96.7% (+32.3 points) | 77.8% → 91.1% (+13.3 points) | +| Overall | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 100.0% (+2.8 points) | 100.0% → 94.4% (-5.6 points) | +| Correctness | 61.1% → 94.4% (+33.3 points) | 74.4% → 93.3% (+18.9 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 37.1% → 93.8% (+56.7 points) | 58.8% → 94.2% (+35.4 points) | -| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 64.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 36.5% → 96.5% (+60.0 points) | 56.5% → 100.0% (+43.5 points) | +| Efficiency | 79.2% — baseline ran, but no comparable score was available; uplift unavailable | 63.8% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -97,20 +99,12 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 96.2% (-2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.4% → 93.5% (-4.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.3% → 88.2% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | - -### Tier Status - -| Tier | Purpose | Status | Evidence | -|---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | -| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | +| codex | Efficiency | Token-only comparison | 90.3% → 88.3% (-2.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Skill Version(s):
-0.3.0 (source: changelog, released 2025-05-21)
+0.3.0 (source: changelog, released 2025-05-21; git SHA f73b929, committed 2026-11-05)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig index 100ab5f98..7b07f0a29 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJkMzNkNGU2NWE1NDAwN2EzMDJkNDg2M2IxNDgzYzkzMDA4YTBmOTRkZDAzNGIzMTZkZGIwZTE5MmU1ZjZjMGQ3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYzVmMjM3OWUxZTdhMTAyNjhlNzEzMGMzNTk2N2Y1OTBjMTkxNzdiZDQ5OGQ4ZWJjZTY5NjU2YzA5ZDQwNmNlNCIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzQ0MmM1YTA3NTE2NTc3YWJhN2U5OGQ2MjkyMzE2MmI4YWY2ZmM1OTNkNmZmYmFlMWRmMzFjZDU2Njk3ZjRmZSIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IiwKICAgICAgICAibmFtZSI6ICJhZ2VudHMvb3BlbmFpLnlhbWwiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzZTQwYjc0ZjM3MWZkMzAzMDIzODI3MDE0MWE2N2FiMmM2OGEzNTJjODNmYzcxNmRkOTQ0MjcwODA3M2M0NzVhIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMTFjNGNhOWUxMWZjMzZjMDQwYjFlMjgwZGRkYmNiZTg5ZGI0ZmExMTIwOGI2Y2UzNjU4YjJlYzkyNjE2N2U0ZiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9oZXJtZXMtY2F0YWxvZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImEyNGI1OTRiMDZmYTRlMTZjODkzN2RhNTRiOWJmYzJjYmVjZThiY2M2NmI5ZDAyYTY3Mjk0MzQwMjFhMTY4NDgiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdGF4b25vbXktcm91dGluZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjBhMDkzN2Y3YjA3ZTViMWVkYzhjMjdiNzBmZjg2NWM2OTg3NzZkNGFmNTU4ZTcxYjQxMGFkYTlhZDJiMDI1ZWUiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDsS+e34JG8dwuEyIgealbx6MuWJKxHK1JCOy4R0w9zy4pvixnIHDB7ATbKlkFX3XUCMD3DtNLaf4gBFLqsCfmamscklrLP1nc6uYTQhVvgPMpiPoFLPsg6OBtJXTE2uRtuYQ==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIzN2VhYzVjZGI0YTQ4ZjkwMzIwNTA2MTE3Y2YzN2EwNDgwOGZjYTIyMzVkNzBhOWE3ODQwYzk2NDU4N2NmNWJiIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiMDZlZGM2MjZjYWFhMjFkZTAwYThhNGQ1ZWY4YzA3YWEyZTkyZGYyMGE5ZDQ2NzgwMTIxMzI4Mzc2YjBjZWI5NCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI0MmQzNjQxNjQzYzE0Mjc5MjhiYWJjYzcyNmUwNDkxMTI5YTE0MDI5Y2RiYWZiYmFiZWJjZDI3ZDMyMTI3YzJkIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFnZW50cy9vcGVuYWkueWFtbCIsCiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiM2U0MGI3NGYzNzFmZDMwMzAyMzgyNzAxNDFhNjdhYjJjNjhhMzUyYzgzZmM3MTZkZDk0NDI3MDgwNzNjNDc1YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2hlcm1lcy1jYXRhbG9nLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjUyZTlhMDlkYjFkM2M2OWFjYWRlMDlkYjAwYjM1ZWE0ZThmZjRlNmRiZmE2MWMzNGVjOTQ5MzZlZDI2MGUyNDEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy90YXhvbm9teS1yb3V0aW5nLm1kIiwKICAgICAgICAiZGlnZXN0IjogImEyNGI1OTRiMDZmYTRlMTZjODkzN2RhNTRiOWJmYzJjYmVjZThiY2M2NmI5ZDAyYTY3Mjk0MzQwMjFhMTY4NDgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI2YjRhN2EzOWM1MTgwMjY2ZTQyOTM2ZWI0YWEzOWViOWIxMTQ0NTBiYjI0MGUyMGZhNjRlYWUzZDJjMTk4NWQwIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCshSRlacq2acqbjwIXAlyBkc1/Dta+FVLwWtPtTDUq7bny5B3t9k0NP0SschnuidsCMGULEBMMyIQ+SSGPZifmrGcBl9QCDVqswA4rDMIOhul2ig36unoxrNRfKikYjAlhBg==","keyid":""}]}} \ No newline at end of file diff --git a/skills/nvidia-skill-finder/BENCHMARK.md b/skills/nvidia-skill-finder/BENCHMARK.md index 263e29911..78e6dc28b 100644 --- a/skills/nvidia-skill-finder/BENCHMARK.md +++ b/skills/nvidia-skill-finder/BENCHMARK.md @@ -10,8 +10,8 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `nvidia-skill-finder` - Validation status: `passed` -- Evaluation date: 2026-11-04 -- Evaluator version: `1.6.6` +- Evaluation date: 2026-11-05 +- Evaluator version: `1.6.8` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 18 evaluation tasks (13 positive, 5 negative) - Dataset digest: `sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281` (skill-evaluator-dataset-snapshot/1) @@ -28,11 +28,11 @@ SkillSpector provider, model, analyzer revision, and determinism evidence retain | Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | | --- | --- | --- | --- | --- | --- | --- | -| nvidia-skill-finder | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | -| nvidia-skill-finder | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | -| nvidia-skill-finder | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | -| nvidia-skill-finder | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | -| nvidia-skill-finder | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.3` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | - Controls for `nvidia-skill-finder`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 97.2% (-2.8 points) | 97.2% → 91.7% (-5.5 points) | -| Correctness | 64.4% → 96.7% (+32.3 points) | 77.8% → 91.1% (+13.3 points) | +| Overall | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | 87.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 97.2% (-2.8 points) | 94.4% → 86.1% (-8.3 points) | +| Correctness | 65.6% → 96.7% (+31.1 points) | 76.7% → 95.6% (+18.9 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 37.1% → 93.8% (+56.7 points) | 58.8% → 94.2% (+35.4 points) | -| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 64.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 35.1% → 96.5% (+61.4 points) | 61.6% → 97.2% (+35.6 points) | +| Efficiency | 78.7% — baseline ran, but no comparable score was available; uplift unavailable | 63.4% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 96.2% (-2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 96.3% (-2.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.3% → 88.2% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.5% → 88.3% (-1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,45 +80,45 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 2,940,059 | 1,733,228 | +1,206,831 | +69.63% | skill 18/18; base 18/18 | -| claude-code | nvidia-skill-finder-diff-bare-invocation | 159,556 | 107,051 | +52,505 | +49.05% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 160,640 | 106,788 | +53,852 | +50.43% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-deep-research | 349,223 | 35,273 | +313,950 | +890.06% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-express-route | 185,632 | 223,103 | -37,471 | -16.80% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 72,345 | 71,645 | +700 | +0.98% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-python-refactor | 142,972 | 141,956 | +1,016 | +0.72% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-react-optimize | 145,032 | 142,851 | +2,181 | +1.53% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-video-editing | 107,845 | 71,165 | +36,680 | +51.54% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-cad-to-simready | 154,290 | 143,642 | +10,648 | +7.41% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dicom | 154,191 | 35,577 | +118,614 | +333.40% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dynamo-kv | 231,606 | 144,237 | +87,369 | +60.57% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,338 | 108,293 | +45,045 | +41.60% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,754 | 72,321 | +82,433 | +113.98% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,792 | 35,680 | +118,112 | +331.03% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,387 | 71,488 | +81,899 | +114.56% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,938 | 151,301 | +3,637 | +2.40% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,414 | 35,400 | +118,014 | +333.37% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,104 | 35,457 | +117,647 | +331.80% | skill 1/1; base 1/1 | -| codex | All cases | 1,208,315 | 806,320 | +401,995 | +49.86% | skill 18/18; base 18/18 | -| codex | nvidia-skill-finder-diff-bare-invocation | 86,372 | 35,913 | +50,459 | +140.50% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-browse-no-keywords | 93,587 | 47,277 | +46,310 | +97.95% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-deep-research | 80,712 | 187,624 | -106,912 | -56.98% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-express-route | 40,930 | 40,522 | +408 | +1.01% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,679 | 27,181 | +14,498 | +53.34% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-python-refactor | 40,995 | 40,331 | +664 | +1.65% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-react-optimize | 69,618 | 40,882 | +28,736 | +70.29% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-video-editing | 13,444 | 13,257 | +187 | +1.41% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-cad-to-simready | 67,899 | 13,327 | +54,572 | +409.48% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dicom | 126,618 | 57,984 | +68,634 | +118.37% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dynamo-kv | 71,047 | 17,627 | +53,420 | +303.06% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-gpu-pandas | 53,320 | 26,172 | +27,148 | +103.73% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-isaac-lab | 103,549 | 52,434 | +51,115 | +97.48% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-driver-install | 61,973 | 17,645 | +44,328 | +251.22% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-setup | 53,437 | 17,828 | +35,609 | +199.74% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-multigpu-training | 78,115 | 66,339 | +11,776 | +17.75% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-openusd-optimization | 54,639 | 65,106 | -10,467 | -16.08% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-vehicle-routing | 70,381 | 38,871 | +31,510 | +81.06% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 4,148,374 | 2,539,548 | +1,608,826 | +63.35% | skill 36/36; base 36/36 | +| claude-code | All cases | 2,756,252 | 1,731,091 | +1,025,161 | +59.22% | skill 18/18; base 18/18 | +| claude-code | nvidia-skill-finder-diff-bare-invocation | 158,059 | 107,030 | +51,029 | +47.68% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 164,344 | 70,831 | +93,513 | +132.02% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-deep-research | 194,159 | 35,239 | +158,920 | +450.98% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-express-route | 187,002 | 221,909 | -34,907 | -15.73% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 108,498 | 143,628 | -35,130 | -24.46% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-python-refactor | 107,980 | 141,827 | -33,847 | -23.86% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-react-optimize | 108,155 | 106,957 | +1,198 | +1.12% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-video-editing | 108,223 | 107,431 | +792 | +0.74% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-cad-to-simready | 153,854 | 106,990 | +46,864 | +43.80% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dicom | 153,851 | 35,602 | +118,249 | +332.14% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dynamo-kv | 234,195 | 143,882 | +90,313 | +62.77% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,696 | 107,250 | +46,446 | +43.31% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,768 | 37,091 | +117,677 | +317.27% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 154,068 | 35,956 | +118,112 | +328.49% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,631 | 107,202 | +46,429 | +43.31% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,932 | 151,307 | +3,625 | +2.40% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,236 | 35,425 | +117,811 | +332.56% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,601 | 35,534 | +118,067 | +332.26% | skill 1/1; base 1/1 | +| codex | All cases | 1,243,207 | 702,854 | +540,353 | +76.88% | skill 18/18; base 18/18 | +| codex | nvidia-skill-finder-diff-bare-invocation | 65,133 | 39,985 | +25,148 | +62.89% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-browse-no-keywords | 78,147 | 48,458 | +29,689 | +61.27% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-deep-research | 102,367 | 48,821 | +53,546 | +109.68% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-express-route | 41,069 | 40,479 | +590 | +1.46% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,698 | 41,283 | +415 | +1.01% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-python-refactor | 41,350 | 40,372 | +978 | +2.42% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-react-optimize | 41,009 | 82,437 | -41,428 | -50.25% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-video-editing | 40,857 | 13,264 | +27,593 | +208.03% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-cad-to-simready | 67,947 | 13,334 | +54,613 | +409.58% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dicom | 74,682 | 30,159 | +44,523 | +147.63% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dynamo-kv | 71,009 | 45,191 | +25,818 | +57.13% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-gpu-pandas | 68,996 | 21,810 | +47,186 | +216.35% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-isaac-lab | 146,943 | 67,402 | +79,541 | +118.01% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-driver-install | 73,345 | 17,678 | +55,667 | +314.89% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-setup | 53,502 | 17,751 | +35,751 | +201.40% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-multigpu-training | 57,962 | 30,505 | +27,457 | +90.01% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-openusd-optimization | 68,939 | 67,955 | +984 | +1.45% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-vehicle-routing | 108,252 | 35,970 | +72,282 | +200.95% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 3,999,459 | 2,433,945 | +1,565,514 | +64.32% | skill 36/36; base 36/36 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -126,7 +126,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | @@ -138,9 +138,9 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nvidia-skill-finder/SKILL.md`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:74`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:103`) -- **MEDIUM** SECURITY/Security finding (SQP-2): Because the skill triggers on many NVIDIA-related requests without being asked, the agent may download and run remote np (`SKILL.md:109`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:113`) -- 11 additional finding(s) are available in the full evaluation artifacts. +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:135`) +- 9 additional finding(s) are available in the full evaluation artifacts. diff --git a/skills/nvidia-skill-finder/skill-card.md b/skills/nvidia-skill-finder/skill-card.md index 4e8547b49..d73a38b4a 100644 --- a/skills/nvidia-skill-finder/skill-card.md +++ b/skills/nvidia-skill-finder/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill; it checks the live NVIDIA skills catalog and recommends matching skills, while staying quiet for generic non-NVIDIA route, optimize, deploy, AI, video, data, or infrastructure tasks.
+Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill, by checking the live NVIDIA skills catalog and recommending (and, with approval, installing) matching skills.
This skill is ready for commercial/non-commercial use.
@@ -7,9 +7,9 @@ This skill is ready for commercial/non-commercial use.
NVIDIA
### License/Terms of Use:
-CC-BY-4.0 AND Apache-2.0
+CC-BY-4.0 AND Apache 2.0
## Use Case:
-Developers and engineers working with NVIDIA products, hardware, SDKs, and services use this skill to discover, evaluate, and (with explicit approval) install relevant NVIDIA agent skills from the live catalog, including Hermes Agent catalog lookups.
+Developers and engineers using coding agents (Claude Code, Codex, Hermes) use this skill to discover, install, and start using NVIDIA agent skills for NVIDIA products, hardware, SDKs, and workflows such as CUDA, NIM, NeMo, Omniverse, Isaac, RAPIDS, cuOpt, Jetson, and TensorRT.
### Deployment Geography for Use:
Global
@@ -30,13 +30,14 @@ Mitigation: Review and scan skill before deployment.
- [NVIDIA Skills catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
- [NVIDIA Skills on build.nvidia.com](https://build.nvidia.com/skills)
- [NVIDIA Skills catalog index (skills.sh.json)](https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json)
+- [NVIDIA Skills marketplace metadata](https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json)
## Skill Output:
**Output Type(s):** [Analysis, Shell commands]
**Output Format:** [Markdown with inline bash code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Recommends at most three skills ordered by confidence; installs only after explicit user approval.]
+**Other Properties Related to Output:** [Recommends at most three skills, ordered by confidence; installs only after explicit user approval.]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -45,7 +46,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-18 evaluation tasks (13 positive, 5 negative), 1 attempt per task, run in the `k8s-sandbox` environment with each task attempt in its own isolated sandbox pod; each task was also attempted without the skill as a baseline.
+18 evaluation tasks (13 positive, 5 negative), 1 attempt per task, each run in its own isolated sandbox pod (`k8s-sandbox` environment); each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -56,13 +57,13 @@ Reported benchmark dimensions:
- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Accuracy: final-answer correctness against the reference answer.
-- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
-- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
-- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
@@ -77,12 +78,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.4% — baseline ran, but no comparable score was available; uplift unavailable | 87.4% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 97.2% (-2.8 points) | 97.2% → 91.7% (-5.5 points) | -| Correctness | 64.4% → 96.7% (+32.3 points) | 77.8% → 91.1% (+13.3 points) | +| Overall | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | 87.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 97.2% (-2.8 points) | 94.4% → 86.1% (-8.3 points) | +| Correctness | 65.6% → 96.7% (+31.1 points) | 76.7% → 95.6% (+18.9 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 37.1% → 93.8% (+56.7 points) | 58.8% → 94.2% (+35.4 points) | -| Efficiency | 79.6% — baseline ran, but no comparable score was available; uplift unavailable | 64.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 35.1% → 96.5% (+61.4 points) | 61.6% → 97.2% (+35.6 points) | +| Efficiency | 78.7% — baseline ran, but no comparable score was available; uplift unavailable | 63.4% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -97,15 +98,15 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 96.2% (-2.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.3% → 96.3% (-2.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.3% → 88.2% (-1.1 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.5% → 88.3% (-1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | -### Tier Status +#### Tier Status | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | diff --git a/skills/nvidia-skill-finder/skill.oms.sig b/skills/nvidia-skill-finder/skill.oms.sig index 100ab5f98..f6f39c6a7 100644 --- a/skills/nvidia-skill-finder/skill.oms.sig +++ b/skills/nvidia-skill-finder/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJkMzNkNGU2NWE1NDAwN2EzMDJkNDg2M2IxNDgzYzkzMDA4YTBmOTRkZDAzNGIzMTZkZGIwZTE5MmU1ZjZjMGQ3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYzVmMjM3OWUxZTdhMTAyNjhlNzEzMGMzNTk2N2Y1OTBjMTkxNzdiZDQ5OGQ4ZWJjZTY5NjU2YzA5ZDQwNmNlNCIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzQ0MmM1YTA3NTE2NTc3YWJhN2U5OGQ2MjkyMzE2MmI4YWY2ZmM1OTNkNmZmYmFlMWRmMzFjZDU2Njk3ZjRmZSIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IiwKICAgICAgICAibmFtZSI6ICJhZ2VudHMvb3BlbmFpLnlhbWwiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzZTQwYjc0ZjM3MWZkMzAzMDIzODI3MDE0MWE2N2FiMmM2OGEzNTJjODNmYzcxNmRkOTQ0MjcwODA3M2M0NzVhIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMTFjNGNhOWUxMWZjMzZjMDQwYjFlMjgwZGRkYmNiZTg5ZGI0ZmExMTIwOGI2Y2UzNjU4YjJlYzkyNjE2N2U0ZiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9oZXJtZXMtY2F0YWxvZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImEyNGI1OTRiMDZmYTRlMTZjODkzN2RhNTRiOWJmYzJjYmVjZThiY2M2NmI5ZDAyYTY3Mjk0MzQwMjFhMTY4NDgiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdGF4b25vbXktcm91dGluZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjBhMDkzN2Y3YjA3ZTViMWVkYzhjMjdiNzBmZjg2NWM2OTg3NzZkNGFmNTU4ZTcxYjQxMGFkYTlhZDJiMDI1ZWUiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDsS+e34JG8dwuEyIgealbx6MuWJKxHK1JCOy4R0w9zy4pvixnIHDB7ATbKlkFX3XUCMD3DtNLaf4gBFLqsCfmamscklrLP1nc6uYTQhVvgPMpiPoFLPsg6OBtJXTE2uRtuYQ==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI1ZjI4ZTg1MzNiMTlhMzU5NmFhYmJkOTliMzhmYzgzMTBiOGM3M2EyNGE4MTgyOTdlNDJiMTFiZWY3ODA3Zjk3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJjYjgwZWNkOGUyOWZjOWU0OWNmMzVjMzc1YTNmZDg4YTFjZjMwYmFjOGYwYzM5YmRmMTk1OWI0YjQzMjRlZTliIiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI0MmQzNjQxNjQzYzE0Mjc5MjhiYWJjYzcyNmUwNDkxMTI5YTE0MDI5Y2RiYWZiYmFiZWJjZDI3ZDMyMTI3YzJkIiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImYyYjI5NDIyNTIwNjEyNTA1NjQzYzZhMWRkMmY0NDFiNGExYmYzNTBmYTUxYzZmZTZlZjQ1YTUyNmE3MmI3MzkiLAogICAgICAgICJuYW1lIjogImFnZW50cy9vcGVuYWkueWFtbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjNlNDBiNzRmMzcxZmQzMDMwMjM4MjcwMTQxYTY3YWIyYzY4YTM1MmM4M2ZjNzE2ZGQ5NDQyNzA4MDczYzQ3NWEiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI1MmU5YTA5ZGIxZDNjNjlhY2FkZTA5ZGIwMGIzNWVhNGU4ZmY0ZTZkYmZhNjFjMzRlYzk0OTM2ZWQyNjBlMjQxIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2hlcm1lcy1jYXRhbG9nLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYTI0YjU5NGIwNmZhNGUxNmM4OTM3ZGE1NGI5YmZjMmNiZWNlOGJjYzY2YjlkMDJhNjcyOTQzNDAyMWExNjg0OCIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy90YXhvbm9teS1yb3V0aW5nLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzI3MmEzMWQ0NDEzNTkzOGJmODU4YzkxZWZlOGI4NDIwNTRiNjc0OTY4YjQ0MTAwNjc3NWNiMjFmMWRjYjc3YyIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDurwcR9Uo7ZdB1sswMakmZVWesQ/JepOl/MgSJzhb7uAHm2UNACTxepeSJ5PnfkGUCMBwmwMNnY5g3XfG7eaVC70ekITQ0ql64HaVzuTBM5mEhE99945gNWyU3vSqXPPY5Hw==","keyid":""}]}} \ No newline at end of file From cc8c77975867b1296b4e2d6cf67ae94071c98816 Mon Sep 17 00:00:00 2001 From: Jason Dudash Date: Tue, 10 Nov 2026 07:39:41 -0500 Subject: [PATCH 16/19] fix(nvidia-skill-finder): make the no-jq catalog fallback portable The python3 -c one-liner was broken two ways. Backslash escapes inside an f-string expression are only legal from Python 3.12, so it raised SyntaxError on macOS's system 3.9 and Ubuntu 22.04's 3.10. Separately, Hermes's approval layer classifies inline interpreter scripts as "script execution via -e/-c flag" and refuses to run them at all. Use python3 -m json.tool piped through grep and cut instead, which works unchanged on 3.9 and clears the same approval check, and tell the model not to reach for python3 -c. Thanks to @mosheabr for catching the version incompatibility in review. Signed-off-by: Jason Dudash --- .../nvidia-skill-finder/references/hermes-catalog.md | 11 +++++++++-- .../nvidia-skill-finder/references/hermes-catalog.md | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md index 957e572bb..30eb8026f 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -17,8 +17,15 @@ curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/ma | jq -r '.skills[] | "\(.name): \(.description[0:110])"' ``` -If `jq` is unavailable, pipe the same URL through -`python3 -c 'import json,sys; [print(f"{s[\"name\"]}: {s[\"description\"][:110]}") for s in json.load(sys.stdin)["skills"]]'`. +If `jq` is unavailable, use this instead. Do not reach for `python3 -c`: Hermes +blocks inline interpreter scripts as "script execution via -e/-c flag", and the +nested quoting such a one-liner needs is a syntax error on Python 3.11 and +older. + +```bash +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ + | python3 -m json.tool | grep -E '"(name|description)":' | cut -c1-130 +``` Read the list and pick the best match on the descriptions. Prefer this over searching. Searching looks cheaper but is not: it matches the query as a plain diff --git a/skills/nvidia-skill-finder/references/hermes-catalog.md b/skills/nvidia-skill-finder/references/hermes-catalog.md index 957e572bb..30eb8026f 100644 --- a/skills/nvidia-skill-finder/references/hermes-catalog.md +++ b/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -17,8 +17,15 @@ curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/ma | jq -r '.skills[] | "\(.name): \(.description[0:110])"' ``` -If `jq` is unavailable, pipe the same URL through -`python3 -c 'import json,sys; [print(f"{s[\"name\"]}: {s[\"description\"][:110]}") for s in json.load(sys.stdin)["skills"]]'`. +If `jq` is unavailable, use this instead. Do not reach for `python3 -c`: Hermes +blocks inline interpreter scripts as "script execution via -e/-c flag", and the +nested quoting such a one-liner needs is a syntax error on Python 3.11 and +older. + +```bash +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ + | python3 -m json.tool | grep -E '"(name|description)":' | cut -c1-130 +``` Read the list and pick the best match on the descriptions. Prefer this over searching. Searching looks cheaper but is not: it matches the query as a plain From 5120ecb61e8bb246a16b3eab10366e4a1390b44b Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Tue, 10 Nov 2026 13:32:44 +0000 Subject: [PATCH 17/19] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- .../skills/nvidia-skill-finder/BENCHMARK.md | 104 +++++++++--------- .../skills/nvidia-skill-finder/skill-card.md | 49 +++++---- .../skills/nvidia-skill-finder/skill.oms.sig | 2 +- skills/nvidia-skill-finder/BENCHMARK.md | 102 ++++++++--------- skills/nvidia-skill-finder/skill-card.md | 44 ++++---- skills/nvidia-skill-finder/skill.oms.sig | 2 +- 6 files changed, 154 insertions(+), 149 deletions(-) diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md index d6acadca1..6833cc221 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md @@ -10,8 +10,8 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `nvidia-skill-finder` - Validation status: `passed` -- Evaluation date: 2026-11-05 -- Evaluator version: `1.6.8` +- Evaluation date: 2026-11-10 +- Evaluator version: `1.6.9` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 18 evaluation tasks (13 positive, 5 negative) - Dataset digest: `sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281` (skill-evaluator-dataset-snapshot/1) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 97.2% → 100.0% (+2.8 points) | 100.0% → 94.4% (-5.6 points) | -| Correctness | 61.1% → 94.4% (+33.3 points) | 74.4% → 93.3% (+18.9 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 36.5% → 96.5% (+60.0 points) | 56.5% → 100.0% (+43.5 points) | -| Efficiency | 79.2% — baseline ran, but no comparable score was available; uplift unavailable | 63.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | 87.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 100.0% (+2.8 points) | 97.2% → 91.7% (-5.5 points) | +| Correctness | 63.3% → 96.7% (+33.4 points) | 77.8% → 93.3% (+15.5 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 37.1% → 96.5% (+59.4 points) | 59.5% → 94.4% (+34.9 points) | +| Efficiency | 81.8% — baseline ran, but no comparable score was available; uplift unavailable | 65.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.4% → 93.5% (-4.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.5% → 96.3% (-2.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.3% → 88.3% (-2.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.4% → 89.2% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,45 +80,45 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 2,563,597 | 1,758,243 | +805,354 | +45.80% | skill 18/18; base 18/18 | -| claude-code | nvidia-skill-finder-diff-bare-invocation | 163,746 | 143,042 | +20,704 | +14.47% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 161,047 | 141,786 | +19,261 | +13.58% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-deep-research | 153,874 | 35,301 | +118,573 | +335.89% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-express-route | 71,576 | 220,779 | -149,203 | -67.58% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 107,929 | 107,725 | +204 | +0.19% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-python-refactor | 70,858 | 106,486 | -35,628 | -33.46% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-react-optimize | 108,001 | 106,758 | +1,243 | +1.16% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-video-editing | 107,860 | 71,136 | +36,724 | +51.63% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-cad-to-simready | 153,429 | 107,060 | +46,369 | +43.31% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dicom | 153,857 | 35,665 | +118,192 | +331.39% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dynamo-kv | 233,877 | 144,043 | +89,834 | +62.37% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,226 | 107,361 | +45,865 | +42.72% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,854 | 72,468 | +82,386 | +113.69% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,460 | 35,709 | +117,751 | +329.75% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,472 | 107,158 | +46,314 | +43.22% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-multigpu-training | 155,620 | 109,524 | +46,096 | +42.09% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,832 | 70,893 | +82,939 | +116.99% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,079 | 35,349 | +117,730 | +333.05% | skill 1/1; base 1/1 | -| codex | All cases | 1,203,358 | 676,098 | +527,260 | +77.99% | skill 18/18; base 18/18 | -| codex | nvidia-skill-finder-diff-bare-invocation | 68,816 | 39,893 | +28,923 | +72.50% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-browse-no-keywords | 52,454 | 78,017 | -25,563 | -32.77% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-deep-research | 86,657 | 36,197 | +50,460 | +139.40% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-express-route | 41,182 | 54,920 | -13,738 | -25.01% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,660 | 27,124 | +14,536 | +53.59% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-python-refactor | 41,277 | 40,337 | +940 | +2.33% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-react-optimize | 41,082 | 68,215 | -27,133 | -39.78% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-video-editing | 13,432 | 13,353 | +79 | +0.59% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-cad-to-simready | 67,885 | 13,304 | +54,581 | +410.26% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dicom | 74,804 | 30,361 | +44,443 | +146.38% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dynamo-kv | 71,043 | 41,039 | +30,004 | +73.11% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-gpu-pandas | 69,037 | 22,219 | +46,818 | +210.71% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-isaac-lab | 156,536 | 52,466 | +104,070 | +198.36% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-driver-install | 73,541 | 17,722 | +55,819 | +314.97% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-setup | 53,484 | 17,795 | +35,689 | +200.56% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-multigpu-training | 73,674 | 22,164 | +51,510 | +232.40% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-openusd-optimization | 68,956 | 65,081 | +3,875 | +5.95% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-vehicle-routing | 107,838 | 35,891 | +71,947 | +200.46% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 3,766,955 | 2,434,341 | +1,332,614 | +54.74% | skill 36/36; base 36/36 | +| claude-code | All cases | 2,711,937 | 1,615,131 | +1,096,806 | +67.91% | skill 18/18; base 18/18 | +| claude-code | nvidia-skill-finder-diff-bare-invocation | 158,444 | 70,725 | +87,719 | +124.03% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 159,838 | 70,753 | +89,085 | +125.91% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-deep-research | 159,845 | 35,205 | +124,640 | +354.04% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-express-route | 225,059 | 183,972 | +41,087 | +22.33% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 108,443 | 107,258 | +1,185 | +1.10% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-python-refactor | 70,868 | 178,071 | -107,203 | -60.20% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-react-optimize | 145,172 | 143,073 | +2,099 | +1.47% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-video-editing | 108,172 | 71,109 | +37,063 | +52.12% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-cad-to-simready | 153,814 | 106,978 | +46,836 | +43.78% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dicom | 153,832 | 70,874 | +82,958 | +117.05% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dynamo-kv | 191,996 | 144,429 | +47,567 | +32.93% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,323 | 108,233 | +45,090 | +41.66% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,640 | 37,221 | +117,419 | +315.46% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,527 | 35,927 | +117,600 | +327.33% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,277 | 71,735 | +81,542 | +113.67% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,876 | 108,772 | +46,104 | +42.39% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,774 | 35,357 | +118,417 | +334.92% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,037 | 35,439 | +117,598 | +331.83% | skill 1/1; base 1/1 | +| codex | All cases | 1,183,079 | 708,839 | +474,240 | +66.90% | skill 18/18; base 18/18 | +| codex | nvidia-skill-finder-diff-bare-invocation | 82,443 | 39,981 | +42,462 | +106.21% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-browse-no-keywords | 74,467 | 86,422 | -11,955 | -13.83% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-deep-research | 74,436 | 58,872 | +15,564 | +26.44% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-express-route | 40,960 | 54,898 | -13,938 | -25.39% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,583 | 27,181 | +14,402 | +52.99% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-python-refactor | 41,193 | 40,300 | +893 | +2.22% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-react-optimize | 55,211 | 40,942 | +14,269 | +34.85% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-video-editing | 13,438 | 13,250 | +188 | +1.42% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-cad-to-simready | 52,176 | 17,520 | +34,656 | +197.81% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dicom | 71,768 | 30,224 | +41,544 | +137.45% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dynamo-kv | 55,355 | 44,962 | +10,393 | +23.12% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-gpu-pandas | 68,989 | 17,840 | +51,149 | +286.71% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-isaac-lab | 99,264 | 69,570 | +29,694 | +42.68% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-driver-install | 69,382 | 17,671 | +51,711 | +292.63% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-setup | 69,017 | 17,581 | +51,436 | +292.57% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-multigpu-training | 57,797 | 30,750 | +27,047 | +87.96% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-openusd-optimization | 145,160 | 65,166 | +79,994 | +122.75% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-vehicle-routing | 70,440 | 35,709 | +34,731 | +97.26% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 3,895,016 | 2,323,970 | +1,571,046 | +67.60% | skill 36/36; base 36/36 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -126,7 +126,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | @@ -138,9 +138,9 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:74`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:103`) -- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:113`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:135`) -- 9 additional finding(s) are available in the full evaluation artifacts. +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:113`) +- 11 additional finding(s) are available in the full evaluation artifacts. diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md index 6e9f9d813..ffcbb7ccd 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill; it checks the live NVIDIA skills catalog and recommends matching NVIDIA skills (installing only with user approval), while staying quiet for generic non-NVIDIA route, optimize, deploy, AI, video, data, or infrastructure tasks.
+Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill; it detects NVIDIA relevance, checks the live NVIDIA skills catalog, and recommends (and, with user approval, installs) matching skills.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
CC-BY-4.0 AND Apache-2.0
## Use Case:
-Developers and engineers using AI coding agents (such as Claude Code, Codex, or Hermes Agent) use this skill to discover, recommend, and—with explicit approval—install NVIDIA agent skills relevant to NVIDIA products, SDKs, hardware, and workflows such as CUDA, NIM, NeMo, Omniverse/OpenUSD, Isaac, Jetson, RAPIDS, cuOpt, Dynamo, Holoscan, TensorRT, DeepStream, and VSS.
+Developers and engineers using AI coding agents (Claude Code, Codex, Hermes Agent) use this skill to discover, install, and start using NVIDIA agent skills relevant to NVIDIA products, SDKs, hardware, and workflows such as CUDA, NIM, NeMo, Omniverse, Isaac, RAPIDS, Jetson, and TensorRT.
### Deployment Geography for Use:
Global
@@ -30,15 +30,14 @@ Mitigation: Review and scan skill before deployment.
- [NVIDIA Skills catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
- [NVIDIA Skills on build.nvidia.com](https://build.nvidia.com/skills)
- [NVIDIA Skills catalog index (skills.sh.json)](https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json)
-- [NVIDIA Skills marketplace metadata (metadata.json)](https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json)
-- [NVIDIA Skills documentation](https://docs.nvidia.com/skills)
+- [NVIDIA Skills marketplace metadata](https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json)
## Skill Output:
-**Output Type(s):** [Analysis, Shell commands, Configuration instructions]
-**Output Format:** [Markdown with inline bash code blocks (skill recommendations with name, rationale, install command, and first prompt)]
+**Output Type(s):** [Analysis, Shell commands]
+**Output Format:** [Markdown with inline bash code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Recommends at most three skills ordered by confidence; installs only after explicit user approval.]
+**Other Properties Related to Output:** [Recommends at most three skills ordered by confidence; install commands are run only after explicit user approval.]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -47,7 +46,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-18 evaluation tasks (13 positive, 5 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment), with each task also run as a no-skill baseline. Evaluator version `1.6.8`; evaluation date 2026-11-05.
+18 evaluation tasks (13 positive, 5 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment), evaluator version `1.6.9`, evaluated 2026-11-10. Each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -69,22 +68,20 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
-> ✅ **Overall verdict: PASS — Recommended for publication** +**Overall verdict:** ✅ PASS — Recommended for publication -### Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. ### Results at a Glance | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 94.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 97.2% → 100.0% (+2.8 points) | 100.0% → 94.4% (-5.6 points) | -| Correctness | 61.1% → 94.4% (+33.3 points) | 74.4% → 93.3% (+18.9 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 36.5% → 96.5% (+60.0 points) | 56.5% → 100.0% (+43.5 points) | -| Efficiency | 79.2% — baseline ran, but no comparable score was available; uplift unavailable | 63.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | 87.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 100.0% (+2.8 points) | 97.2% → 91.7% (-5.5 points) | +| Correctness | 63.3% → 96.7% (+33.4 points) | 77.8% → 93.3% (+15.5 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 37.1% → 96.5% (+59.4 points) | 59.5% → 94.4% (+34.9 points) | +| Efficiency | 81.8% — baseline ran, but no comparable score was available; uplift unavailable | 65.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -92,19 +89,27 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. -#### Comparison details +### Comparison details Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.4% → 93.5% (-4.9 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.5% → 96.3% (-2.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 90.3% → 88.3% (-2.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 89.4% → 89.2% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | ## Skill Version(s):
-0.3.0 (source: changelog, released 2025-05-21; git SHA f73b929, committed 2026-11-05)
+0.3.0 (source: changelog, released 2025-05-21)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig index 7b07f0a29..9a9c10912 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIzN2VhYzVjZGI0YTQ4ZjkwMzIwNTA2MTE3Y2YzN2EwNDgwOGZjYTIyMzVkNzBhOWE3ODQwYzk2NDU4N2NmNWJiIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiMDZlZGM2MjZjYWFhMjFkZTAwYThhNGQ1ZWY4YzA3YWEyZTkyZGYyMGE5ZDQ2NzgwMTIxMzI4Mzc2YjBjZWI5NCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI0MmQzNjQxNjQzYzE0Mjc5MjhiYWJjYzcyNmUwNDkxMTI5YTE0MDI5Y2RiYWZiYmFiZWJjZDI3ZDMyMTI3YzJkIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFnZW50cy9vcGVuYWkueWFtbCIsCiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiM2U0MGI3NGYzNzFmZDMwMzAyMzgyNzAxNDFhNjdhYjJjNjhhMzUyYzgzZmM3MTZkZDk0NDI3MDgwNzNjNDc1YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2hlcm1lcy1jYXRhbG9nLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjUyZTlhMDlkYjFkM2M2OWFjYWRlMDlkYjAwYjM1ZWE0ZThmZjRlNmRiZmE2MWMzNGVjOTQ5MzZlZDI2MGUyNDEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy90YXhvbm9teS1yb3V0aW5nLm1kIiwKICAgICAgICAiZGlnZXN0IjogImEyNGI1OTRiMDZmYTRlMTZjODkzN2RhNTRiOWJmYzJjYmVjZThiY2M2NmI5ZDAyYTY3Mjk0MzQwMjFhMTY4NDgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI2YjRhN2EzOWM1MTgwMjY2ZTQyOTM2ZWI0YWEzOWViOWIxMTQ0NTBiYjI0MGUyMGZhNjRlYWUzZDJjMTk4NWQwIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCshSRlacq2acqbjwIXAlyBkc1/Dta+FVLwWtPtTDUq7bny5B3t9k0NP0SschnuidsCMGULEBMMyIQ+SSGPZifmrGcBl9QCDVqswA4rDMIOhul2ig36unoxrNRfKikYjAlhBg==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJjMDQ3OTViYTVmZDY2YjA1YzI1YjIxNzNiZjJlNWJjMGE0MjU1NDNhZTVlMDk5Y2ViMjQ5MDBlYTJlZTcyOWMzIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMzg0YmUyMjMxNThiMDYyMzM4NWQxMGVkYmU3OThlMTE5N2NiNDA0NzM1NjA4NGRlN2QwNmUxZGQ1NjcwNjk5MSIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNDJkMzY0MTY0M2MxNDI3OTI4YmFiY2M3MjZlMDQ5MTEyOWExNDAyOWNkYmFmYmJhYmViY2QyN2QzMjEyN2MyZCIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IiwKICAgICAgICAibmFtZSI6ICJhZ2VudHMvb3BlbmFpLnlhbWwiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIzZTQwYjc0ZjM3MWZkMzAzMDIzODI3MDE0MWE2N2FiMmM2OGEzNTJjODNmYzcxNmRkOTQ0MjcwODA3M2M0NzVhIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNzZmYTAxYWQwY2JjZDJjODA4Y2FjOTlkM2Y1NjE1MDRlNjc5MDI3MzY2NTk1MDAwNDk0Yzk1NDA2YWI4NDM0MyIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9oZXJtZXMtY2F0YWxvZy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImEyNGI1OTRiMDZmYTRlMTZjODkzN2RhNTRiOWJmYzJjYmVjZThiY2M2NmI5ZDAyYTY3Mjk0MzQwMjFhMTY4NDgiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdGF4b25vbXktcm91dGluZy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjQwNzJkNTEwODZiN2VjOWU0OGNmOGYzODFjMDA4NmVmNTVkNTBiZDc4YTczYzBkYjc5ZjBmYjY3ODljYjVkYjUiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMGeb+1J68GpUK7yjxF1Vdmh1NQQajbyRRqOVFJqElpTUeDoFS0GaQu2/RJDE3JascQIwXyHauj4fBvKl3btU7gdf4eMzwMckwuY4lnNdyZ9GBD1Uxf4DV3JeTU5/DVMxQ+H6","keyid":""}]}} \ No newline at end of file diff --git a/skills/nvidia-skill-finder/BENCHMARK.md b/skills/nvidia-skill-finder/BENCHMARK.md index 78e6dc28b..280edff7f 100644 --- a/skills/nvidia-skill-finder/BENCHMARK.md +++ b/skills/nvidia-skill-finder/BENCHMARK.md @@ -10,8 +10,8 @@ Recommended for publication based on the completed evaluation evidence in this r - Skill: `nvidia-skill-finder` - Validation status: `passed` -- Evaluation date: 2026-11-05 -- Evaluator version: `1.6.8` +- Evaluation date: 2026-11-10 +- Evaluator version: `1.6.9` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 18 evaluation tasks (13 positive, 5 negative) - Dataset digest: `sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281` (skill-evaluator-dataset-snapshot/1) @@ -50,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | 87.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 97.2% (-2.8 points) | 94.4% → 86.1% (-8.3 points) | -| Correctness | 65.6% → 96.7% (+31.1 points) | 76.7% → 95.6% (+18.9 points) | +| Overall | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 97.2% (±0.0 points) | 100.0% → 91.7% (-8.3 points) | +| Correctness | 64.4% → 96.7% (+32.3 points) | 74.4% → 96.7% (+22.3 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 35.1% → 96.5% (+61.4 points) | 61.6% → 97.2% (+35.6 points) | -| Efficiency | 78.7% — baseline ran, but no comparable score was available; uplift unavailable | 63.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 36.4% → 95.8% (+59.4 points) | 58.6% → 98.6% (+40.0 points) | +| Efficiency | 81.3% — baseline ran, but no comparable score was available; uplift unavailable | 64.4% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -70,9 +70,9 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 96.3% (-2.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 96.4% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.5% → 88.3% (-1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.3% → 88.9% (-1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | ## Token Usage @@ -80,45 +80,45 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 2,756,252 | 1,731,091 | +1,025,161 | +59.22% | skill 18/18; base 18/18 | -| claude-code | nvidia-skill-finder-diff-bare-invocation | 158,059 | 107,030 | +51,029 | +47.68% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 164,344 | 70,831 | +93,513 | +132.02% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-deep-research | 194,159 | 35,239 | +158,920 | +450.98% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-express-route | 187,002 | 221,909 | -34,907 | -15.73% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 108,498 | 143,628 | -35,130 | -24.46% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-python-refactor | 107,980 | 141,827 | -33,847 | -23.86% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-react-optimize | 108,155 | 106,957 | +1,198 | +1.12% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-video-editing | 108,223 | 107,431 | +792 | +0.74% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-cad-to-simready | 153,854 | 106,990 | +46,864 | +43.80% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dicom | 153,851 | 35,602 | +118,249 | +332.14% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dynamo-kv | 234,195 | 143,882 | +90,313 | +62.77% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,696 | 107,250 | +46,446 | +43.31% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,768 | 37,091 | +117,677 | +317.27% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 154,068 | 35,956 | +118,112 | +328.49% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,631 | 107,202 | +46,429 | +43.31% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,932 | 151,307 | +3,625 | +2.40% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,236 | 35,425 | +117,811 | +332.56% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,601 | 35,534 | +118,067 | +332.26% | skill 1/1; base 1/1 | -| codex | All cases | 1,243,207 | 702,854 | +540,353 | +76.88% | skill 18/18; base 18/18 | -| codex | nvidia-skill-finder-diff-bare-invocation | 65,133 | 39,985 | +25,148 | +62.89% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-browse-no-keywords | 78,147 | 48,458 | +29,689 | +61.27% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-deep-research | 102,367 | 48,821 | +53,546 | +109.68% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-express-route | 41,069 | 40,479 | +590 | +1.46% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,698 | 41,283 | +415 | +1.01% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-python-refactor | 41,350 | 40,372 | +978 | +2.42% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-react-optimize | 41,009 | 82,437 | -41,428 | -50.25% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-video-editing | 40,857 | 13,264 | +27,593 | +208.03% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-cad-to-simready | 67,947 | 13,334 | +54,613 | +409.58% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dicom | 74,682 | 30,159 | +44,523 | +147.63% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dynamo-kv | 71,009 | 45,191 | +25,818 | +57.13% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-gpu-pandas | 68,996 | 21,810 | +47,186 | +216.35% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-isaac-lab | 146,943 | 67,402 | +79,541 | +118.01% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-driver-install | 73,345 | 17,678 | +55,667 | +314.89% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-setup | 53,502 | 17,751 | +35,751 | +201.40% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-multigpu-training | 57,962 | 30,505 | +27,457 | +90.01% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-openusd-optimization | 68,939 | 67,955 | +984 | +1.45% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-vehicle-routing | 108,252 | 35,970 | +72,282 | +200.95% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 3,999,459 | 2,433,945 | +1,565,514 | +64.32% | skill 36/36; base 36/36 | +| claude-code | All cases | 2,745,616 | 1,880,457 | +865,159 | +46.01% | skill 18/18; base 18/18 | +| claude-code | nvidia-skill-finder-diff-bare-invocation | 158,578 | 107,454 | +51,124 | +47.58% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 160,580 | 71,104 | +89,476 | +125.84% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-deep-research | 153,995 | 35,277 | +118,718 | +336.53% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-express-route | 149,387 | 221,515 | -72,128 | -32.56% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 108,200 | 143,204 | -35,004 | -24.44% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-python-refactor | 143,192 | 141,724 | +1,468 | +1.04% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-react-optimize | 144,879 | 106,899 | +37,980 | +35.53% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-video-editing | 107,858 | 107,020 | +838 | +0.78% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-cad-to-simready | 153,258 | 143,526 | +9,732 | +6.78% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dicom | 153,494 | 35,674 | +117,820 | +330.27% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dynamo-kv | 233,649 | 144,374 | +89,275 | +61.84% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-gpu-pandas | 154,289 | 107,412 | +46,877 | +43.64% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,854 | 72,465 | +82,389 | +113.69% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,954 | 35,683 | +118,271 | +331.45% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,470 | 107,265 | +46,205 | +43.08% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,720 | 193,877 | -39,157 | -20.20% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-openusd-optimization | 154,034 | 70,646 | +83,388 | +118.04% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,225 | 35,338 | +117,887 | +333.60% | skill 1/1; base 1/1 | +| codex | All cases | 1,217,904 | 634,064 | +583,840 | +92.08% | skill 18/18; base 18/18 | +| codex | nvidia-skill-finder-diff-bare-invocation | 81,560 | 35,508 | +46,052 | +129.69% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-browse-no-keywords | 85,718 | 74,582 | +11,136 | +14.93% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-deep-research | 86,248 | 41,228 | +45,020 | +109.20% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-express-route | 41,182 | 54,623 | -13,441 | -24.61% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-generic-kubernetes | 70,198 | 41,207 | +28,991 | +70.35% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-python-refactor | 40,807 | 40,749 | +58 | +0.14% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-react-optimize | 41,069 | 40,473 | +596 | +1.47% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-video-editing | 13,401 | 13,277 | +124 | +0.93% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-cad-to-simready | 67,896 | 13,288 | +54,608 | +410.96% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dicom | 110,655 | 23,439 | +87,216 | +372.10% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dynamo-kv | 55,280 | 17,709 | +37,571 | +212.16% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-gpu-pandas | 69,086 | 22,158 | +46,928 | +211.79% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-isaac-lab | 99,343 | 49,006 | +50,337 | +102.72% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-driver-install | 73,473 | 17,650 | +55,823 | +316.28% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-setup | 68,993 | 17,646 | +51,347 | +290.98% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-multigpu-training | 73,752 | 22,455 | +51,297 | +228.44% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-openusd-optimization | 68,934 | 65,372 | +3,562 | +5.45% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-vehicle-routing | 70,309 | 43,694 | +26,615 | +60.91% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 3,963,520 | 2,514,521 | +1,448,999 | +57.63% | skill 36/36; base 36/36 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. @@ -126,7 +126,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 15 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | @@ -138,9 +138,9 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nvidia-skill-finder/SKILL.md`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:74`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:103`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:183`) - **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:113`) -- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:135`) -- 9 additional finding(s) are available in the full evaluation artifacts. +- 10 additional finding(s) are available in the full evaluation artifacts. diff --git a/skills/nvidia-skill-finder/skill-card.md b/skills/nvidia-skill-finder/skill-card.md index d73a38b4a..99e251036 100644 --- a/skills/nvidia-skill-finder/skill-card.md +++ b/skills/nvidia-skill-finder/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill, by checking the live NVIDIA skills catalog and recommending (and, with approval, installing) matching skills.
+Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill; it triggers on NVIDIA products, hardware, software, SDKs, GPUs, Jetson/JetPack/L4T/BSP/SDK Manager/driver/flashing/setup, CUDA, NIM, NeMo, Omniverse/OpenUSD/SimReady, Isaac, RAPIDS/cuDF, cuPyNumeric, cuOpt, Dynamo, Holoscan, TensorRT, DeepStream, VSS, TAO, and NGC/NVCF, and does not apply to generic non-NVIDIA route, optimize, deploy, AI, video, data, or infrastructure tasks.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
CC-BY-4.0 AND Apache 2.0
## Use Case:
-Developers and engineers using coding agents (Claude Code, Codex, Hermes) use this skill to discover, install, and start using NVIDIA agent skills for NVIDIA products, hardware, SDKs, and workflows such as CUDA, NIM, NeMo, Omniverse, Isaac, RAPIDS, cuOpt, Jetson, and TensorRT.
+Developers and engineers using AI coding agents (such as Claude Code, Codex, or Hermes Agent) use this skill to discover, recommend, and, with explicit user approval, install NVIDIA skills from the live NVIDIA skills catalog that match an NVIDIA product, task, or workflow.
### Deployment Geography for Use:
Global
@@ -34,10 +34,10 @@ Mitigation: Review and scan skill before deployment.
## Skill Output:
-**Output Type(s):** [Analysis, Shell commands]
+**Output Type(s):** [Skill recommendations, Shell commands]
**Output Format:** [Markdown with inline bash code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Recommends at most three skills, ordered by confidence; installs only after explicit user approval.]
+**Other Properties Related to Output:** [Recommends at most three skills ordered by confidence; install commands are run only after explicit user approval.]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
@@ -46,7 +46,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-18 evaluation tasks (13 positive, 5 negative), 1 attempt per task, each run in its own isolated sandbox pod (`k8s-sandbox` environment); each task was also attempted without the skill as a baseline.
+18 evaluation tasks (13 positive, 5 negative); 1 attempt per task, each run in its own isolated sandbox pod (`k8s-sandbox` environment); each task was also attempted without the skill as a baseline. Evaluator version `1.6.9`; dataset digest `sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281` (skill-evaluator-dataset-snapshot/1).
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -57,13 +57,13 @@ Reported benchmark dimensions:
- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `skill_efficiency`: Tool-call productivity (legacy wire id; routing is scored under Discoverability).
-- `accuracy`: Final-answer correctness against the reference answer.
-- `goal_accuracy`: Whether the user's goal was achieved.
-- `behavior_check`: Whether the expected workflow behavior was followed.
-- `token_efficiency`: Actual uncached prompt plus completion usage (50% of Efficiency).
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
@@ -78,12 +78,12 @@ Recommended for publication based on the completed evaluation evidence in this r | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | 87.5% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 100.0% → 97.2% (-2.8 points) | 94.4% → 86.1% (-8.3 points) | -| Correctness | 65.6% → 96.7% (+31.1 points) | 76.7% → 95.6% (+18.9 points) | +| Overall | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 97.2% (±0.0 points) | 100.0% → 91.7% (-8.3 points) | +| Correctness | 64.4% → 96.7% (+32.3 points) | 74.4% → 96.7% (+22.3 points) | | Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 35.1% → 96.5% (+61.4 points) | 61.6% → 97.2% (+35.6 points) | -| Efficiency | 78.7% — baseline ran, but no comparable score was available; uplift unavailable | 63.4% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 36.4% → 95.8% (+59.4 points) | 58.6% → 98.6% (+40.0 points) | +| Efficiency | 81.3% — baseline ran, but no comparable score was available; uplift unavailable | 64.4% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -98,20 +98,20 @@ Shared-source comparisons use the same signals and weights on both conditions. T | Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | | --- | --- | --- | --- | --- | | claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| claude-code | Efficiency | Token-only comparison | 98.3% → 96.3% (-2.0 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 96.4% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | | codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | -| codex | Efficiency | Token-only comparison | 89.5% → 88.3% (-1.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Efficiency | Token-only comparison | 90.3% → 88.9% (-1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | -#### Tier Status +### Tier Status | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 14 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 15 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | ## Skill Version(s):
-0.3.0 (source: changelog, released 2025-05-21)
+0.3.0 (source: changelog, released 2025-05-21; git commit cc8c779)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/nvidia-skill-finder/skill.oms.sig b/skills/nvidia-skill-finder/skill.oms.sig index f6f39c6a7..4a6e12ff3 100644 --- a/skills/nvidia-skill-finder/skill.oms.sig +++ b/skills/nvidia-skill-finder/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI1ZjI4ZTg1MzNiMTlhMzU5NmFhYmJkOTliMzhmYzgzMTBiOGM3M2EyNGE4MTgyOTdlNDJiMTFiZWY3ODA3Zjk3IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJjYjgwZWNkOGUyOWZjOWU0OWNmMzVjMzc1YTNmZDg4YTFjZjMwYmFjOGYwYzM5YmRmMTk1OWI0YjQzMjRlZTliIiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI0MmQzNjQxNjQzYzE0Mjc5MjhiYWJjYzcyNmUwNDkxMTI5YTE0MDI5Y2RiYWZiYmFiZWJjZDI3ZDMyMTI3YzJkIiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImYyYjI5NDIyNTIwNjEyNTA1NjQzYzZhMWRkMmY0NDFiNGExYmYzNTBmYTUxYzZmZTZlZjQ1YTUyNmE3MmI3MzkiLAogICAgICAgICJuYW1lIjogImFnZW50cy9vcGVuYWkueWFtbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjNlNDBiNzRmMzcxZmQzMDMwMjM4MjcwMTQxYTY3YWIyYzY4YTM1MmM4M2ZjNzE2ZGQ5NDQyNzA4MDczYzQ3NWEiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI1MmU5YTA5ZGIxZDNjNjlhY2FkZTA5ZGIwMGIzNWVhNGU4ZmY0ZTZkYmZhNjFjMzRlYzk0OTM2ZWQyNjBlMjQxIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2hlcm1lcy1jYXRhbG9nLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYTI0YjU5NGIwNmZhNGUxNmM4OTM3ZGE1NGI5YmZjMmNiZWNlOGJjYzY2YjlkMDJhNjcyOTQzNDAyMWExNjg0OCIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy90YXhvbm9teS1yb3V0aW5nLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzI3MmEzMWQ0NDEzNTkzOGJmODU4YzkxZWZlOGI4NDIwNTRiNjc0OTY4YjQ0MTAwNjc3NWNiMjFmMWRjYjc3YyIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDurwcR9Uo7ZdB1sswMakmZVWesQ/JepOl/MgSJzhb7uAHm2UNACTxepeSJ5PnfkGUCMBwmwMNnY5g3XfG7eaVC70ekITQ0ql64HaVzuTBM5mEhE99945gNWyU3vSqXPPY5Hw==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI5Nzg0OTEyM2QwMjY5NjI1MTZhYTlhODJhYzhhYzNiZTgxODM3ZTNiOTM5ODhhMzRlNjZjMDM2NWQ1ZTcxNzIxIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJmZmRiM2YwMTUwZTQ0MDQzZGM5ODU1OWM1NzE1NzU3YzdhNmMyZTM1YzJhZGI4YzlkMzY4NTI5ZjQzZjM3MDNmIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjQyZDM2NDE2NDNjMTQyNzkyOGJhYmNjNzI2ZTA0OTExMjlhMTQwMjljZGJhZmJiYWJlYmNkMjdkMzIxMjdjMmQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiYWdlbnRzL29wZW5haS55YW1sIiwKICAgICAgICAiZGlnZXN0IjogImYyYjI5NDIyNTIwNjEyNTA1NjQzYzZhMWRkMmY0NDFiNGExYmYzNTBmYTUxYzZmZTZlZjQ1YTUyNmE3MmI3MzkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICIzZTQwYjc0ZjM3MWZkMzAzMDIzODI3MDE0MWE2N2FiMmM2OGEzNTJjODNmYzcxNmRkOTQ0MjcwODA3M2M0NzVhIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvaGVybWVzLWNhdGFsb2cubWQiLAogICAgICAgICJkaWdlc3QiOiAiNzZmYTAxYWQwY2JjZDJjODA4Y2FjOTlkM2Y1NjE1MDRlNjc5MDI3MzY2NTk1MDAwNDk0Yzk1NDA2YWI4NDM0MyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3RheG9ub215LXJvdXRpbmcubWQiLAogICAgICAgICJkaWdlc3QiOiAiYTI0YjU5NGIwNmZhNGUxNmM4OTM3ZGE1NGI5YmZjMmNiZWNlOGJjYzY2YjlkMDJhNjcyOTQzNDAyMWExNjg0OCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogImM4Yzc1NGI2YTZlNDlhMzEwZTRjMDM2YjljNjc0MmM2MmYzNWIxM2YxNzlhNjhkMzE0ODM5NzdmZjNjY2U0ZWYiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMAqQa0apK17qXOL2hm+jiqAoKkVbKLViZXBYTbY/J7wenFy9ZKIlW4BEqiTSdBKvIwIwRNpMQ7amt07GRT7cQZrScF7A1lzem+PTyJJkFUIHH2u0SRM0wyY+cFPmaYUkfaYL","keyid":""}]}} \ No newline at end of file From 4649ce2f9ebcfc58a8cac64bf72a8ad18b1a5be5 Mon Sep 17 00:00:00 2001 From: Moshe Abramovitch Date: Tue, 10 Nov 2026 09:40:50 -0600 Subject: [PATCH 18/19] chore(benchmarks): refresh after signature attach Co-Authored-By: Claude Opus 5.5 Signed-off-by: Moshe Abramovitch --- benchmarks.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/benchmarks.json b/benchmarks.json index 994b43657..e4b68a979 100644 --- a/benchmarks.json +++ b/benchmarks.json @@ -6025,9 +6025,9 @@ }, { "skill": "nvidia-skill-finder", - "evaluation_date": "2026-11-04", + "evaluation_date": "2026-11-10", "environment": "k8s-sandbox", - "evaluator_version": "1.6.6", + "evaluator_version": "1.6.9", "dataset_digest": "sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281", "validation_status": "passed", "tasks": 18, @@ -6041,7 +6041,7 @@ "catalog_dir": "nvidia-skill-finder", "component": "NVIDIA Skills", "has_results": true, - "average_uplift_pct": 21.57 + "average_uplift_pct": 24.28 }, { "skill": "nvshmem-collect-performance-data", @@ -34187,7 +34187,7 @@ "num": null, "agent": "claude-code", "score_pct": 97.2, - "uplift_pct": -2.8 + "uplift_pct": 0.0 }, { "catalog_dir": "nvidia-skill-finder", @@ -34196,7 +34196,7 @@ "num": null, "agent": "codex", "score_pct": 91.7, - "uplift_pct": -5.5 + "uplift_pct": -8.3 }, { "catalog_dir": "nvidia-skill-finder", @@ -34213,8 +34213,8 @@ "dimension": "Correctness", "num": null, "agent": "codex", - "score_pct": 91.1, - "uplift_pct": 13.3 + "score_pct": 96.7, + "uplift_pct": 22.3 }, { "catalog_dir": "nvidia-skill-finder", @@ -34240,8 +34240,8 @@ "dimension": "Effectiveness", "num": null, "agent": "claude-code", - "score_pct": 93.8, - "uplift_pct": 56.7 + "score_pct": 95.8, + "uplift_pct": 59.4 }, { "catalog_dir": "nvidia-skill-finder", @@ -34249,8 +34249,8 @@ "dimension": "Effectiveness", "num": null, "agent": "codex", - "score_pct": 94.2, - "uplift_pct": 35.4 + "score_pct": 98.6, + "uplift_pct": 40.0 }, { "catalog_dir": "nvidia-skill-finder", @@ -34258,7 +34258,7 @@ "dimension": "Efficiency", "num": null, "agent": "claude-code", - "score_pct": 79.6, + "score_pct": 81.3, "uplift_pct": null }, { @@ -34267,7 +34267,7 @@ "dimension": "Efficiency", "num": null, "agent": "codex", - "score_pct": 64.7, + "score_pct": 64.4, "uplift_pct": null }, { From 7dbac471057fc4e9ab5495a5db53714f41b5b45d Mon Sep 17 00:00:00 2001 From: Moshe Abramovitch Date: Tue, 10 Nov 2026 09:42:21 -0600 Subject: [PATCH 19/19] chore(versions): refresh after signature attach Co-Authored-By: Claude Opus 5.5 Signed-off-by: Moshe Abramovitch --- versions.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/versions.json b/versions.json index e44d2e0aa..49eae5547 100644 --- a/versions.json +++ b/versions.json @@ -2118,9 +2118,9 @@ { "name": "nvidia-skill-finder", "path": "skills/nvidia-skill-finder", - "content_digest": "sha256:b5c517e02618ea17df54fe204435adff5bcbd65716d56f4f8a5ec97eced30a0d", - "last_commit": "6ae4b5e52e84a5d446bb278023700cedc880dd23", - "last_modified": "2026-11-05" + "content_digest": "sha256:63a07ba8ba0d12c9cb85c8a5dc4dabd52ec4fc56c7cb933a340aee85cb5724a2", + "last_commit": "5120ecb61e8bb246a16b3eab10366e4a1390b44b", + "last_modified": "2026-11-10" }, { "name": "nvshmem-collect-performance-data",