diff --git a/benchmarks.json b/benchmarks.json index bed118c15..e4b68a979 100644 --- a/benchmarks.json +++ b/benchmarks.json @@ -6025,13 +6025,13 @@ }, { "skill": "nvidia-skill-finder", - "evaluation_date": "2026-10-01", + "evaluation_date": "2026-11-10", "environment": "k8s-sandbox", - "evaluator_version": "1.5.6", - "dataset_digest": "sha256:df412e229dfed3687e1112c8f5bc439a6f3695291f3d357f049002621e90845f", - "validation_status": null, + "evaluator_version": "1.6.9", + "dataset_digest": "sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281", + "validation_status": "passed", "tasks": 18, - "attempts_per_task": 3, + "attempts_per_task": 1, "pass_threshold_pct": null, "verdict": "PASS", "agents": [ @@ -6041,7 +6041,7 @@ "catalog_dir": "nvidia-skill-finder", "component": "NVIDIA Skills", "has_results": true, - "average_uplift_pct": 26.97 + "average_uplift_pct": 24.28 }, { "skill": "nvshmem-collect-performance-data", @@ -34187,7 +34187,7 @@ "num": null, "agent": "claude-code", "score_pct": 97.2, - "uplift_pct": -1.0 + "uplift_pct": 0.0 }, { "catalog_dir": "nvidia-skill-finder", @@ -34195,8 +34195,8 @@ "dimension": "Security", "num": null, "agent": "codex", - "score_pct": 94.4, - "uplift_pct": 5.9 + "score_pct": 91.7, + "uplift_pct": -8.3 }, { "catalog_dir": "nvidia-skill-finder", @@ -34205,7 +34205,7 @@ "num": null, "agent": "claude-code", "score_pct": 96.7, - "uplift_pct": 27.1 + "uplift_pct": 32.3 }, { "catalog_dir": "nvidia-skill-finder", @@ -34213,8 +34213,8 @@ "dimension": "Correctness", "num": null, "agent": "codex", - "score_pct": 100.0, - "uplift_pct": 29.2 + "score_pct": 96.7, + "uplift_pct": 22.3 }, { "catalog_dir": "nvidia-skill-finder", @@ -34231,7 +34231,7 @@ "dimension": "Discoverability", "num": null, "agent": "codex", - "score_pct": 94.6, + "score_pct": 95.0, "uplift_pct": null }, { @@ -34240,8 +34240,8 @@ "dimension": "Effectiveness", "num": null, "agent": "claude-code", - "score_pct": 87.9, - "uplift_pct": 51.8 + "score_pct": 95.8, + "uplift_pct": 59.4 }, { "catalog_dir": "nvidia-skill-finder", @@ -34249,8 +34249,8 @@ "dimension": "Effectiveness", "num": null, "agent": "codex", - "score_pct": 89.7, - "uplift_pct": 48.8 + "score_pct": 98.6, + "uplift_pct": 40.0 }, { "catalog_dir": "nvidia-skill-finder", @@ -34258,7 +34258,7 @@ "dimension": "Efficiency", "num": null, "agent": "claude-code", - "score_pct": 78.4, + "score_pct": 81.3, "uplift_pct": null }, { @@ -34267,7 +34267,7 @@ "dimension": "Efficiency", "num": null, "agent": "codex", - "score_pct": 66.5, + "score_pct": 64.4, "uplift_pct": null }, { diff --git a/plugins.d/nvidia-skills.yml b/plugins.d/nvidia-skills.yml index d33b939e0..58fb55498 100644 --- a/plugins.d/nvidia-skills.yml +++ b/plugins.d/nvidia-skills.yml @@ -1,6 +1,6 @@ name: nvidia-skills -version: "1.4.1" +version: "1.5.0" description: Find the right NVIDIA skill for GPU acceleration, CUDA, AI agents, data loading, training, inference, robotics, Physical AI, Omniverse, simulation, and other NVIDIA-powered workflows. display_name: NVIDIA Skills diff --git a/plugins/nvidia-skills/.claude-plugin/plugin.json b/plugins/nvidia-skills/.claude-plugin/plugin.json index a90ea7076..3fd8a3947 100644 --- a/plugins/nvidia-skills/.claude-plugin/plugin.json +++ b/plugins/nvidia-skills/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "nvidia-skills", - "version": "1.4.1", + "version": "1.5.0", "description": "Find the right NVIDIA skill for GPU acceleration, CUDA, AI agents, data loading, training, inference, robotics, Physical AI, Omniverse, simulation, and other NVIDIA-powered workflows.", "displayName": "NVIDIA Skills", "author": { diff --git a/plugins/nvidia-skills/.codex-plugin/plugin.json b/plugins/nvidia-skills/.codex-plugin/plugin.json index b1121adfa..37fc83004 100644 --- a/plugins/nvidia-skills/.codex-plugin/plugin.json +++ b/plugins/nvidia-skills/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "nvidia-skills", - "version": "1.4.1", + "version": "1.5.0", "description": "Find the right NVIDIA skill for GPU acceleration, CUDA, AI agents, data loading, training, inference, robotics, Physical AI, Omniverse, simulation, and other NVIDIA-powered workflows.", "author": { "name": "NVIDIA", diff --git a/plugins/nvidia-skills/.cursor-plugin/plugin.json b/plugins/nvidia-skills/.cursor-plugin/plugin.json index 266302a38..50afe2c44 100644 --- a/plugins/nvidia-skills/.cursor-plugin/plugin.json +++ b/plugins/nvidia-skills/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "nvidia-skills", - "version": "1.4.1", + "version": "1.5.0", "description": "Find the right NVIDIA skill for GPU acceleration, CUDA, AI agents, data loading, training, inference, robotics, Physical AI, Omniverse, simulation, and other NVIDIA-powered workflows.", "author": { "name": "NVIDIA" diff --git a/plugins/nvidia-skills/README.md b/plugins/nvidia-skills/README.md index da26fc4cb..f4dea0b84 100644 --- a/plugins/nvidia-skills/README.md +++ b/plugins/nvidia-skills/README.md @@ -4,7 +4,7 @@ This plugin ships one skill, `nvidia-skill-finder`. It checks NVIDIA's live skil ## Availability -This plugin is available as a first-party plugin in the Claude Code, Codex, and Cursor marketplaces. Hermes support is coming soon. +This plugin is available as a first-party plugin in the Claude Code, Codex, Cursor, and Hermes marketplaces. You can also get it by adding the self-hosted `nvidia-official` marketplace from the `NVIDIA/skills` repo. @@ -14,6 +14,13 @@ You can also get it by adding the self-hosted `nvidia-official` marketplace from - **Installs only with approval.** It never installs without asking. When the user agrees, it runs `npx skills add nvidia/skills --skill --global --yes`, which installs that skill from the `main` branch of `NVIDIA/skills` into the agent's global skills directory. - **Wider search.** If nothing in the NVIDIA catalog matches, it may suggest `npx skills find `, which searches skills outside NVIDIA's catalog. +### In Hermes + +Hermes ships its own scanned, profile-aware skills catalog, so the finder uses that instead and never calls `npx`. The details live in the skill's `references/hermes-catalog.md`. + +- **Catalog lookup.** The finder reads the full catalog listing from `raw.githubusercontent.com` and picks from it, rather than searching. Hermes search matches the query as a plain substring, so it misses phrases and truncates broad keywords at a default of 25; reading the whole list once is both cheaper in context and complete. +- **Installs only with approval.** Once a skill is chosen, the finder confirms its identifier with `hermes skills search --source nvidia --json --limit 100` (requires Hermes 0.22+ for `--source`), then installs with `hermes skills install `, which runs Hermes's own security scan and installs into the active profile. Hermes prompts for confirmation itself, so the finder does not pass `--yes`. + The plugin itself contains no code, stores no credentials, and sends no telemetry. The `skills` CLI follows its own policies. ## How this folder is built diff --git a/plugins/nvidia-skills/plugin.json b/plugins/nvidia-skills/plugin.json index 070697bce..40365f865 100644 --- a/plugins/nvidia-skills/plugin.json +++ b/plugins/nvidia-skills/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "nvidia-skills", - "version": "1.4.1", + "version": "1.5.0", "description": "Find the right NVIDIA skill for GPU acceleration, CUDA, AI agents, data loading, training, inference, robotics, Physical AI, Omniverse, simulation, and other NVIDIA-powered workflows.", "author": { "name": "NVIDIA", diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md index 79fd0d79b..6833cc221 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/BENCHMARK.md @@ -9,18 +9,33 @@ Recommended for publication based on the completed evaluation evidence in this r ## Evaluation Metadata - Skill: `nvidia-skill-finder` -- Evaluation date: 2026-10-01 -- Evaluator version: `1.5.6` -- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Validation status: `passed` +- Evaluation date: 2026-11-10 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 18 evaluation tasks (13 positive, 5 negative) -- Dataset digest: `sha256:df412e229dfed3687e1112c8f5bc439a6f3695291f3d357f049002621e90845f` (skill-evaluator-dataset-snapshot/1) -- Attempts per task: 3 +- Dataset digest: `sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 - Environment: `k8s-sandbox` - Tier 2 evidence: required for publication - Tier 3 evidence: required for publication Each task attempt ran in its own isolated sandbox pod. +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| nvidia-skill-finder | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `nvidia-skill-finder`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + ## What This Report Answers The three-tier evaluation checks whether the skill: @@ -35,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 98.2% → 97.2% (-1.0 points) | 88.5% → 94.4% (+5.9 points) | -| Correctness | 69.6% → 96.7% (+27.1 points) | 70.8% → 100.0% (+29.2 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 36.1% → 87.9% (+51.8 points) | 40.9% → 89.7% (+48.8 points) | -| Efficiency | 78.4% — baseline ran, but no comparable score was available; uplift unavailable | 66.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | 87.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 100.0% (+2.8 points) | 97.2% → 91.7% (-5.5 points) | +| Correctness | 63.3% → 96.7% (+33.4 points) | 77.8% → 93.3% (+15.5 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 37.1% → 96.5% (+59.4 points) | 59.5% → 94.4% (+34.9 points) | +| Efficiency | 81.8% — baseline ran, but no comparable score was available; uplift unavailable | 65.9% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -48,59 +63,70 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.5% → 96.3% (-2.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 89.4% → 89.2% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + ## Token Usage Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 2,931,632 | 4,494,280 | N/A | N/A | skill 18/18; base 27/27 | -| claude-code | nvidia-skill-finder-diff-bare-invocation | 175,571 | 151,559 | +24,012 | +15.84% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 136,133 | 120,437 | +15,696 | +13.03% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-deep-research | 170,078 | 363,184 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-neg-express-route | 356,710 | 483,643 | -126,933 | -26.25% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 154,997 | 185,511 | -30,514 | -16.45% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-python-refactor | 120,042 | 119,199 | +843 | +0.71% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-react-optimize | 153,960 | 152,401 | +1,559 | +1.02% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-video-editing | 159,058 | 154,957 | +4,101 | +2.65% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-cad-to-simready | 129,524 | 162,375 | -32,851 | -20.23% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dicom | 177,291 | 157,064 | +20,227 | +12.88% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dynamo-kv | 129,903 | 376,086 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-gpu-pandas | 130,132 | 124,405 | +5,727 | +4.60% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-isaac-lab | 211,676 | 60,437 | +151,239 | +250.24% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 167,216 | 59,583 | N/A | N/A | skill 1/1; base 2/2 | -| claude-code | nvidia-skill-finder-pos-jetson-setup | 130,190 | 335,895 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-multigpu-training | 169,065 | 1,276,278 | -1,107,213 | -86.75% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-openusd-optimization | 129,543 | 89,238 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-vehicle-routing | 130,543 | 122,028 | +8,515 | +6.98% | skill 1/1; base 1/1 | -| codex | All cases | 1,642,585 | 2,128,791 | N/A | N/A | skill 18/18; base 26/26 | -| codex | nvidia-skill-finder-diff-bare-invocation | 73,710 | 685,152 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-diff-browse-no-keywords | 128,756 | 240,494 | -111,738 | -46.46% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-deep-research | 86,599 | 40,331 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-neg-express-route | 177,679 | 69,297 | +108,382 | +156.40% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-generic-kubernetes | 201,213 | 213,286 | -12,073 | -5.66% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-python-refactor | 41,853 | 69,141 | -27,288 | -39.47% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-react-optimize | 98,985 | 127,238 | -28,253 | -22.20% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-video-editing | 69,541 | 55,477 | +14,064 | +25.35% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-cad-to-simready | 74,083 | 25,338 | +48,745 | +192.38% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dicom | 74,117 | 31,845 | +42,272 | +132.74% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dynamo-kv | 94,291 | 307,770 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-pos-gpu-pandas | 71,906 | 18,127 | +53,779 | +296.68% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-isaac-lab | 85,260 | 74,373 | +10,887 | +14.64% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-driver-install | 71,998 | 18,014 | +53,984 | +299.68% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-setup | 71,193 | 66,628 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-pos-multigpu-training | 74,250 | 59,276 | +14,974 | +25.26% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-openusd-optimization | 79,947 | 13,501 | +66,446 | +492.16% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-vehicle-routing | 67,204 | 13,503 | +53,701 | +397.70% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 4,574,217 | 6,623,071 | N/A | N/A | skill 36/36; base 53/53 | - -Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. +| claude-code | All cases | 2,711,937 | 1,615,131 | +1,096,806 | +67.91% | skill 18/18; base 18/18 | +| claude-code | nvidia-skill-finder-diff-bare-invocation | 158,444 | 70,725 | +87,719 | +124.03% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 159,838 | 70,753 | +89,085 | +125.91% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-deep-research | 159,845 | 35,205 | +124,640 | +354.04% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-express-route | 225,059 | 183,972 | +41,087 | +22.33% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 108,443 | 107,258 | +1,185 | +1.10% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-python-refactor | 70,868 | 178,071 | -107,203 | -60.20% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-react-optimize | 145,172 | 143,073 | +2,099 | +1.47% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-video-editing | 108,172 | 71,109 | +37,063 | +52.12% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-cad-to-simready | 153,814 | 106,978 | +46,836 | +43.78% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dicom | 153,832 | 70,874 | +82,958 | +117.05% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dynamo-kv | 191,996 | 144,429 | +47,567 | +32.93% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-gpu-pandas | 153,323 | 108,233 | +45,090 | +41.66% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,640 | 37,221 | +117,419 | +315.46% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,527 | 35,927 | +117,600 | +327.33% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,277 | 71,735 | +81,542 | +113.67% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,876 | 108,772 | +46,104 | +42.39% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-openusd-optimization | 153,774 | 35,357 | +118,417 | +334.92% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,037 | 35,439 | +117,598 | +331.83% | skill 1/1; base 1/1 | +| codex | All cases | 1,183,079 | 708,839 | +474,240 | +66.90% | skill 18/18; base 18/18 | +| codex | nvidia-skill-finder-diff-bare-invocation | 82,443 | 39,981 | +42,462 | +106.21% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-browse-no-keywords | 74,467 | 86,422 | -11,955 | -13.83% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-deep-research | 74,436 | 58,872 | +15,564 | +26.44% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-express-route | 40,960 | 54,898 | -13,938 | -25.39% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-generic-kubernetes | 41,583 | 27,181 | +14,402 | +52.99% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-python-refactor | 41,193 | 40,300 | +893 | +2.22% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-react-optimize | 55,211 | 40,942 | +14,269 | +34.85% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-video-editing | 13,438 | 13,250 | +188 | +1.42% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-cad-to-simready | 52,176 | 17,520 | +34,656 | +197.81% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dicom | 71,768 | 30,224 | +41,544 | +137.45% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dynamo-kv | 55,355 | 44,962 | +10,393 | +23.12% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-gpu-pandas | 68,989 | 17,840 | +51,149 | +286.71% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-isaac-lab | 99,264 | 69,570 | +29,694 | +42.68% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-driver-install | 69,382 | 17,671 | +51,711 | +292.63% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-setup | 69,017 | 17,581 | +51,436 | +292.57% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-multigpu-training | 57,797 | 30,750 | +27,047 | +87.96% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-openusd-optimization | 145,160 | 65,166 | +79,994 | +122.75% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-vehicle-routing | 70,440 | 35,709 | +34,731 | +97.26% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 3,895,016 | 2,323,970 | +1,571,046 | +67.60% | skill 36/36; base 36/36 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. ## Tier Status | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 19 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | @@ -109,12 +135,12 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c
Show detailed findings and successful checks -- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nvidia-skill-finder/SKILL.md`) -- **MEDIUM** SECURITY/Unknown (SQP-1): L032 instructs activation on any "how do I do X" request where X "might be a common task that an NVIDIA skill can help w (`SKILL.md:32`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: The `npx skills add nvidia/skills --list` command is invoked without a pinned package version. When npx resolves an unpi (`SKILL.md:74`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: Same unpinned `npx skills` invocation as above, used as the primary catalog-check mechanism. Without a version pin, any (`SKILL.md:103`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: Another instance of the same unpinned `npx skills` pattern. The cumulative effect of multiple unpinned invocations acros (`SKILL.md:113`) -- 14 additional finding(s) are available in the full evaluation artifacts. +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:74`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:103`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:135`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:113`) +- 11 additional finding(s) are available in the full evaluation artifacts.
diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md index e9aa2f887..3083c8e59 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/SKILL.md @@ -113,7 +113,13 @@ Catalog check means one of: npx skills add nvidia/skills --list ``` -Use the fallback catalog sources only if the CLI is unavailable, blocked, or fails: +- If running in Hermes Agent, do not run `npx`. Follow + [references/hermes-catalog.md](references/hermes-catalog.md) instead. +- Otherwise do not run `hermes` commands. The binary may exist on the machine + even when you are a different agent, and installing through it writes to + Hermes's skills directory, where you cannot load the result. + +Use the fallback catalog sources only if the lookup above is unavailable, blocked, or fails: - https://github.com/NVIDIA/skills/tree/main/skills - https://build.nvidia.com/skills - https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json @@ -149,6 +155,11 @@ If the agent target is unknown, omit `--agent` and let the CLI prompt: npx skills add nvidia/skills --skill --global --yes ``` +- If running in Hermes Agent, install with `hermes skills install` instead so + the skill is security scanned and lands in the active profile. Casing and + identifier rules are in + [references/hermes-catalog.md](references/hermes-catalog.md). + After install, tell the user to restart or reload the agent if their client does not pick up newly installed skills immediately. @@ -163,8 +174,8 @@ does not pick up newly installed skills immediately. Do not fabricate catalog entries or guess at skill names. If catalog lookup fails, say the catalog could not be checked and do not name a concrete slug or -emit an `npx skills add ... --skill ` install command. Offer to share the -catalog URL, retry lookup, or continue with general help. +emit an install command for it, in either form. Offer to share the catalog URL, +retry lookup, or continue with general help. ## When No Skills are Found diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/evals/evals.json b/plugins/nvidia-skills/skills/nvidia-skill-finder/evals/evals.json index 7c351fc94..21fa07091 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/evals/evals.json +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/evals/evals.json @@ -7,7 +7,7 @@ "ground_truth": "The agent identifies Decision Optimization as a strong NVIDIA taxonomy match, checks the live NVIDIA skills catalog, and recommends a current cuOpt routing skill rather than implementing a routing solver from memory.", "expected_behavior": [ "Treats vehicle routing as a strong NVIDIA Decision Optimization signal", - "Checks the live NVIDIA catalog (e.g. `npx skills add nvidia/skills --list`) before naming a specific skill", + "Checks the live NVIDIA catalog before naming a specific skill, using the lookup for the current agent (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`)", "Recommends a cuOpt routing-related skill if present in the catalog", "Asks before installing the skill", "Does not implement a routing solver before offering the skill" @@ -99,7 +99,7 @@ "ground_truth": "The agent treats Jetson driver installation as a strong NVIDIA Robotics / Jetson workflow signal, checks the live NVIDIA catalog before naming a skill, and recommends the current Jetson setup or quick-start skill if available.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Jetson driver / JetPack / BSP request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends a Jetson setup skill (e.g. jetson-quick-start) if present", "Explains that Jetson drivers are normally bundled with JetPack / Jetson Linux rather than installed as standalone desktop drivers", @@ -114,7 +114,7 @@ "ground_truth": "The agent treats Jetson setup as a strong NVIDIA Robotics / Jetson workflow signal, checks the live NVIDIA skills catalog before using product docs as the main answer, and recommends the current Jetson setup or quick-start skill if available.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Jetson setup request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends a Jetson setup skill (e.g. jetson-quick-start) if present", "Provides practical Jetson setup guidance only after the catalog lookup path", @@ -129,7 +129,7 @@ "ground_truth": "The agent treats Isaac Lab as a strong NVIDIA Robotics / Physical AI signal, checks the live NVIDIA skills catalog before giving general Isaac Lab guidance, and recommends a current Isaac-related skill only if the catalog has one.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Isaac Lab request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends an Isaac-related skill if present in the catalog, or says no strong catalog match was found without inventing a skill name", "Asks before installing the skill" diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md new file mode 100644 index 000000000..30eb8026f --- /dev/null +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -0,0 +1,76 @@ +# Hermes Agent: Catalog Lookup and Install + +Follow this instead of the `npx` commands in SKILL.md when you are running in +Hermes Agent. Install goes through Hermes so the skill is security scanned and +lands in the active profile, and `npx` stays unnecessary. + +`--source nvidia` requires Hermes 0.22 or newer; older versions reject the +flag. If it is rejected, drop `--source nvidia` and filter the results for +`NVIDIA/skills` identifiers yourself. + +## Catalog lookup + +List the whole catalog once and choose from it yourself: + +```bash +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ + | jq -r '.skills[] | "\(.name): \(.description[0:110])"' +``` + +If `jq` is unavailable, use this instead. Do not reach for `python3 -c`: Hermes +blocks inline interpreter scripts as "script execution via -e/-c flag", and the +nested quoting such a one-liner needs is a syntax error on Python 3.11 and +older. + +```bash +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ + | python3 -m json.tool | grep -E '"(name|description)":' | cut -c1-130 +``` + +Read the list and pick the best match on the descriptions. Prefer this over +searching. Searching looks cheaper but is not: it matches the query as a plain +case-insensitive substring over each entry's joined fields, so it misses +phrases (`jetson pinmux` returns nothing even though `jetson-customize-pinmux` +exists), and broad keywords return full descriptions for every hit. A handful +of searches costs more context than the complete listing above and can still +miss the right skill. + +Do not use `hermes skills browse` either. It costs more than the listing and +truncates both names and descriptions, so it is worse on accuracy and cost. + +## Confirm the identifier + +Once you have chosen a specific skill, look up that exact name to confirm it is +available in Hermes and to get its identifier: + +```bash +hermes skills search --source nvidia --json --limit 100 +``` + +An exact skill name is a narrow query, so this stays small. Pass `--limit 100` +anyway; the default is 25, which silently truncates broader matches. + +## Install + +Use the `Identifier` from that result, copied verbatim: + +```bash +hermes skills install +``` + +Casing determines trust, not the number of path segments. `NVIDIA/...` resolves +to the trusted first-party source; `nvidia/...` resolves to a community-trust +mirror with a placeholder description. Always use `NVIDIA`. + +The identifier may be a longer path such as `NVIDIA/skills/skills/`; +that form and the short `NVIDIA/skills/` both resolve to the same +trusted skill with the same scan hash. + +Confirm an identifier you are unsure about before installing it: + +```bash +hermes skills inspect +``` + +Hermes prompts for confirmation itself, so do not pass `--yes` on the user's +behalf. diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md index b5487a7ed..ffcbb7ccd 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Helps users discover, install, and start using NVIDIA agent skills by searching the live NVIDIA skills catalog and recommending matches for their task.
+Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill; it detects NVIDIA relevance, checks the live NVIDIA skills catalog, and recommends (and, with user approval, installs) matching skills.
This skill is ready for commercial/non-commercial use.
@@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
CC-BY-4.0 AND Apache-2.0
## Use Case:
-Developers and engineers use this skill to find and install NVIDIA-verified agent skills that match their task, covering domains such as Physical AI, Robotics, Vision AI, Data Science, Inference, Training, Decision Optimization, and GPU Development.
+Developers and engineers using AI coding agents (Claude Code, Codex, Hermes Agent) use this skill to discover, install, and start using NVIDIA agent skills relevant to NVIDIA products, SDKs, hardware, and workflows such as CUDA, NIM, NeMo, Omniverse, Isaac, RAPIDS, Jetson, and TensorRT.
### Deployment Geography for Use:
Global
@@ -25,55 +25,88 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
+- [Hermes Agent: Catalog Lookup and Install](references/hermes-catalog.md)
- [NVIDIA Taxonomy Routing](references/taxonomy-routing.md)
-- [NVIDIA Skills Catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
-- [NVIDIA Skills Catalog (Build Portal)](https://build.nvidia.com/skills)
-- [NVIDIA Agent Skills Documentation](https://docs.nvidia.com/skills)
+- [NVIDIA Skills catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
+- [NVIDIA Skills on build.nvidia.com](https://build.nvidia.com/skills)
+- [NVIDIA Skills catalog index (skills.sh.json)](https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json)
+- [NVIDIA Skills marketplace metadata](https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json)
## Skill Output:
-**Output Type(s):** [Analysis, Shell commands, Configuration instructions]
+**Output Type(s):** [Analysis, Shell commands]
**Output Format:** [Markdown with inline bash code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [None]
+**Other Properties Related to Output:** [Recommends at most three skills ordered by confidence; install commands are run only after explicit user approval.]
## Evaluation Agents Used:
-- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
-- Codex (`openai/openai/gpt-5.5`)
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
## Evaluation Tasks:
-18 evaluation tasks (13 positive, 5 negative), each run with 3 attempts per task in isolated k8s-sandbox pods.
+18 evaluation tasks (13 positive, 5 negative); 1 attempt per task, each in its own isolated sandbox pod (`k8s-sandbox` environment), evaluator version `1.6.9`, evaluated 2026-11-10. Each task was also attempted without the skill as a baseline.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
-- Correctness: Checks final-answer correctness against the reference answer.
-- Discoverability: Checks whether the expected skill was selected and the workflow executed.
-- Effectiveness: Checks whether the user's goal was achieved and the expected workflow behavior was followed.
-- Efficiency: Checks tool-call productivity and token usage efficiency.
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `accuracy`: Final-answer correctness against the reference answer.
-- `goal_accuracy`: Whether the user's goal was achieved.
-- `behavior_check`: Whether the expected workflow behavior was followed.
-- `skill_efficiency`: Tool-call productivity (routing scored under Discoverability).
-- `token_efficiency`: Actual uncached prompt plus completion token usage.
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
-| Dimension | Tasks | Claude Code | Codex | -|---|---:|---:|---:| -| Overall | 18 | 92.0% | 89.0% | -| Security | 18 | 97.2% | 94.4% | -| Correctness | 18 | 96.7% | 100.0% | -| Discoverability | 18 | 100.0% | 94.6% | -| Effectiveness | 18 | 87.9% | 89.7% | -| Efficiency | 18 | 78.4% | 66.5% | +**Overall verdict:** ✅ PASS — Recommended for publication + +**Publication Recommendation:** Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | 87.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 100.0% (+2.8 points) | 97.2% → 91.7% (-5.5 points) | +| Correctness | 63.3% → 96.7% (+33.4 points) | 77.8% → 93.3% (+15.5 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 37.1% → 96.5% (+59.4 points) | 59.5% → 94.4% (+34.9 points) | +| Efficiency | 81.8% — baseline ran, but no comparable score was available; uplift unavailable | 65.9% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.5% → 96.3% (-2.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 89.4% → 89.2% (-0.2 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 16 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | ## Skill Version(s):
0.3.0 (source: changelog, released 2025-05-21)
diff --git a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig index cdde023c3..9a9c10912 100644 --- a/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig +++ b/plugins/nvidia-skills/skills/nvidia-skill-finder/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIwNTM4NDcwMjkyNDZlYjI1NTA2YzMyZjk0MDQ5NDJlOGMyNzM2MzdiYjA1ODI5MzhlMjAwYTU5ZGQ5Mzg0YmE1IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiZWJjN2RlMTc5OGIyYmJhZTE1ZjExOGMyNWVjZjQwOThkYThlN2Q4MmVhYjJhMjhjZWEyNTgwNTA1ZjQwZTBjOSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJmNTllNWMyYzllODljZDM1NjM3NjQyMjUyODBmOWE4ZjA1ZWExMTY0ZWZhOTc4OGZlYTU4OTJmNjViMDRkZjJjIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFnZW50cy9vcGVuYWkueWFtbCIsCiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiNzE2ZmNiM2ZiMTFkMDM2MjYyYTY5NjY0OTI4MmZlYTc2ZmE2MGQyNDIzYTc4MDFlMWQ0YTNlOGY3OTMzZjBlNCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3RheG9ub215LXJvdXRpbmcubWQiLAogICAgICAgICJkaWdlc3QiOiAiYTI0YjU5NGIwNmZhNGUxNmM4OTM3ZGE1NGI5YmZjMmNiZWNlOGJjYzY2YjlkMDJhNjcyOTQzNDAyMWExNjg0OCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjI5OWRkZjA3NzUzOTU3ODY2MzQxMzQ5YjQ5N2MxODQ4MWNmYzI1NWE5MTMxZWYzNTAyNDM4Y2Y3NDA3MTg3OTAiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMGGg1+VbtPZ6EXlNDc0xHBMjfZN0EAGw1oDeoYkA/BuRWHpci0EFllec/wl0PyaTswIxAJLsjn6E8DOrgQvHZgdt+8ILmYrhydTiQ6wLzhXUeJKe3vGsgSFDv0BLA1hZ1kp1ig==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJjMDQ3OTViYTVmZDY2YjA1YzI1YjIxNzNiZjJlNWJjMGE0MjU1NDNhZTVlMDk5Y2ViMjQ5MDBlYTJlZTcyOWMzIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMzg0YmUyMjMxNThiMDYyMzM4NWQxMGVkYmU3OThlMTE5N2NiNDA0NzM1NjA4NGRlN2QwNmUxZGQ1NjcwNjk5MSIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNDJkMzY0MTY0M2MxNDI3OTI4YmFiY2M3MjZlMDQ5MTEyOWExNDAyOWNkYmFmYmJhYmViY2QyN2QzMjEyN2MyZCIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IiwKICAgICAgICAibmFtZSI6ICJhZ2VudHMvb3BlbmFpLnlhbWwiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIzZTQwYjc0ZjM3MWZkMzAzMDIzODI3MDE0MWE2N2FiMmM2OGEzNTJjODNmYzcxNmRkOTQ0MjcwODA3M2M0NzVhIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNzZmYTAxYWQwY2JjZDJjODA4Y2FjOTlkM2Y1NjE1MDRlNjc5MDI3MzY2NTk1MDAwNDk0Yzk1NDA2YWI4NDM0MyIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9oZXJtZXMtY2F0YWxvZy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImEyNGI1OTRiMDZmYTRlMTZjODkzN2RhNTRiOWJmYzJjYmVjZThiY2M2NmI5ZDAyYTY3Mjk0MzQwMjFhMTY4NDgiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvdGF4b25vbXktcm91dGluZy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjQwNzJkNTEwODZiN2VjOWU0OGNmOGYzODFjMDA4NmVmNTVkNTBiZDc4YTczYzBkYjc5ZjBmYjY3ODljYjVkYjUiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMGeb+1J68GpUK7yjxF1Vdmh1NQQajbyRRqOVFJqElpTUeDoFS0GaQu2/RJDE3JascQIwXyHauj4fBvKl3btU7gdf4eMzwMckwuY4lnNdyZ9GBD1Uxf4DV3JeTU5/DVMxQ+H6","keyid":""}]}} \ No newline at end of file diff --git a/skills/nvidia-skill-finder/BENCHMARK.md b/skills/nvidia-skill-finder/BENCHMARK.md index 79fd0d79b..280edff7f 100644 --- a/skills/nvidia-skill-finder/BENCHMARK.md +++ b/skills/nvidia-skill-finder/BENCHMARK.md @@ -9,18 +9,33 @@ Recommended for publication based on the completed evaluation evidence in this r ## Evaluation Metadata - Skill: `nvidia-skill-finder` -- Evaluation date: 2026-10-01 -- Evaluator version: `1.5.6` -- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Validation status: `passed` +- Evaluation date: 2026-11-10 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) - Tasks: 18 evaluation tasks (13 positive, 5 negative) -- Dataset digest: `sha256:df412e229dfed3687e1112c8f5bc439a6f3695291f3d357f049002621e90845f` (skill-evaluator-dataset-snapshot/1) -- Attempts per task: 3 +- Dataset digest: `sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 - Environment: `k8s-sandbox` - Tier 2 evidence: required for publication - Tier 3 evidence: required for publication Each task attempt ran in its own isolated sandbox pod. +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| nvidia-skill-finder | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| nvidia-skill-finder | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `nvidia-skill-finder`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + ## What This Report Answers The three-tier evaluation checks whether the skill: @@ -35,12 +50,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 92.0% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 98.2% → 97.2% (-1.0 points) | 88.5% → 94.4% (+5.9 points) | -| Correctness | 69.6% → 96.7% (+27.1 points) | 70.8% → 100.0% (+29.2 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 36.1% → 87.9% (+51.8 points) | 40.9% → 89.7% (+48.8 points) | -| Efficiency | 78.4% — baseline ran, but no comparable score was available; uplift unavailable | 66.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 97.2% (±0.0 points) | 100.0% → 91.7% (-8.3 points) | +| Correctness | 64.4% → 96.7% (+32.3 points) | 74.4% → 96.7% (+22.3 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 36.4% → 95.8% (+59.4 points) | 58.6% → 98.6% (+40.0 points) | +| Efficiency | 81.3% — baseline ran, but no comparable score was available; uplift unavailable | 64.4% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -48,59 +63,70 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92 A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 96.4% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 90.3% → 88.9% (-1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + ## Token Usage Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 2,931,632 | 4,494,280 | N/A | N/A | skill 18/18; base 27/27 | -| claude-code | nvidia-skill-finder-diff-bare-invocation | 175,571 | 151,559 | +24,012 | +15.84% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 136,133 | 120,437 | +15,696 | +13.03% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-diff-deep-research | 170,078 | 363,184 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-neg-express-route | 356,710 | 483,643 | -126,933 | -26.25% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 154,997 | 185,511 | -30,514 | -16.45% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-python-refactor | 120,042 | 119,199 | +843 | +0.71% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-react-optimize | 153,960 | 152,401 | +1,559 | +1.02% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-neg-video-editing | 159,058 | 154,957 | +4,101 | +2.65% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-cad-to-simready | 129,524 | 162,375 | -32,851 | -20.23% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dicom | 177,291 | 157,064 | +20,227 | +12.88% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-dynamo-kv | 129,903 | 376,086 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-gpu-pandas | 130,132 | 124,405 | +5,727 | +4.60% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-isaac-lab | 211,676 | 60,437 | +151,239 | +250.24% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 167,216 | 59,583 | N/A | N/A | skill 1/1; base 2/2 | -| claude-code | nvidia-skill-finder-pos-jetson-setup | 130,190 | 335,895 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-multigpu-training | 169,065 | 1,276,278 | -1,107,213 | -86.75% | skill 1/1; base 1/1 | -| claude-code | nvidia-skill-finder-pos-openusd-optimization | 129,543 | 89,238 | N/A | N/A | skill 1/1; base 3/3 | -| claude-code | nvidia-skill-finder-pos-vehicle-routing | 130,543 | 122,028 | +8,515 | +6.98% | skill 1/1; base 1/1 | -| codex | All cases | 1,642,585 | 2,128,791 | N/A | N/A | skill 18/18; base 26/26 | -| codex | nvidia-skill-finder-diff-bare-invocation | 73,710 | 685,152 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-diff-browse-no-keywords | 128,756 | 240,494 | -111,738 | -46.46% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-diff-deep-research | 86,599 | 40,331 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-neg-express-route | 177,679 | 69,297 | +108,382 | +156.40% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-generic-kubernetes | 201,213 | 213,286 | -12,073 | -5.66% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-python-refactor | 41,853 | 69,141 | -27,288 | -39.47% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-react-optimize | 98,985 | 127,238 | -28,253 | -22.20% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-neg-video-editing | 69,541 | 55,477 | +14,064 | +25.35% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-cad-to-simready | 74,083 | 25,338 | +48,745 | +192.38% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dicom | 74,117 | 31,845 | +42,272 | +132.74% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-dynamo-kv | 94,291 | 307,770 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-pos-gpu-pandas | 71,906 | 18,127 | +53,779 | +296.68% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-isaac-lab | 85,260 | 74,373 | +10,887 | +14.64% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-driver-install | 71,998 | 18,014 | +53,984 | +299.68% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-jetson-setup | 71,193 | 66,628 | N/A | N/A | skill 1/1; base 3/3 | -| codex | nvidia-skill-finder-pos-multigpu-training | 74,250 | 59,276 | +14,974 | +25.26% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-openusd-optimization | 79,947 | 13,501 | +66,446 | +492.16% | skill 1/1; base 1/1 | -| codex | nvidia-skill-finder-pos-vehicle-routing | 67,204 | 13,503 | +53,701 | +397.70% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 4,574,217 | 6,623,071 | N/A | N/A | skill 36/36; base 53/53 | - -Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. +| claude-code | All cases | 2,745,616 | 1,880,457 | +865,159 | +46.01% | skill 18/18; base 18/18 | +| claude-code | nvidia-skill-finder-diff-bare-invocation | 158,578 | 107,454 | +51,124 | +47.58% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-browse-no-keywords | 160,580 | 71,104 | +89,476 | +125.84% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-diff-deep-research | 153,995 | 35,277 | +118,718 | +336.53% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-express-route | 149,387 | 221,515 | -72,128 | -32.56% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-generic-kubernetes | 108,200 | 143,204 | -35,004 | -24.44% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-python-refactor | 143,192 | 141,724 | +1,468 | +1.04% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-react-optimize | 144,879 | 106,899 | +37,980 | +35.53% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-neg-video-editing | 107,858 | 107,020 | +838 | +0.78% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-cad-to-simready | 153,258 | 143,526 | +9,732 | +6.78% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dicom | 153,494 | 35,674 | +117,820 | +330.27% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-dynamo-kv | 233,649 | 144,374 | +89,275 | +61.84% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-gpu-pandas | 154,289 | 107,412 | +46,877 | +43.64% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-isaac-lab | 154,854 | 72,465 | +82,389 | +113.69% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-driver-install | 153,954 | 35,683 | +118,271 | +331.45% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-jetson-setup | 153,470 | 107,265 | +46,205 | +43.08% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-multigpu-training | 154,720 | 193,877 | -39,157 | -20.20% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-openusd-optimization | 154,034 | 70,646 | +83,388 | +118.04% | skill 1/1; base 1/1 | +| claude-code | nvidia-skill-finder-pos-vehicle-routing | 153,225 | 35,338 | +117,887 | +333.60% | skill 1/1; base 1/1 | +| codex | All cases | 1,217,904 | 634,064 | +583,840 | +92.08% | skill 18/18; base 18/18 | +| codex | nvidia-skill-finder-diff-bare-invocation | 81,560 | 35,508 | +46,052 | +129.69% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-browse-no-keywords | 85,718 | 74,582 | +11,136 | +14.93% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-diff-deep-research | 86,248 | 41,228 | +45,020 | +109.20% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-express-route | 41,182 | 54,623 | -13,441 | -24.61% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-generic-kubernetes | 70,198 | 41,207 | +28,991 | +70.35% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-python-refactor | 40,807 | 40,749 | +58 | +0.14% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-react-optimize | 41,069 | 40,473 | +596 | +1.47% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-neg-video-editing | 13,401 | 13,277 | +124 | +0.93% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-cad-to-simready | 67,896 | 13,288 | +54,608 | +410.96% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dicom | 110,655 | 23,439 | +87,216 | +372.10% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-dynamo-kv | 55,280 | 17,709 | +37,571 | +212.16% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-gpu-pandas | 69,086 | 22,158 | +46,928 | +211.79% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-isaac-lab | 99,343 | 49,006 | +50,337 | +102.72% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-driver-install | 73,473 | 17,650 | +55,823 | +316.28% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-jetson-setup | 68,993 | 17,646 | +51,347 | +290.98% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-multigpu-training | 73,752 | 22,455 | +51,297 | +228.44% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-openusd-optimization | 68,934 | 65,372 | +3,562 | +5.45% | skill 1/1; base 1/1 | +| codex | nvidia-skill-finder-pos-vehicle-routing | 70,309 | 43,694 | +26,615 | +60.91% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 3,963,520 | 2,514,521 | +1,448,999 | +57.63% | skill 36/36; base 36/36 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. ## Tier Status | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 19 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 15 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | @@ -110,11 +136,11 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c Show detailed findings and successful checks - **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nvidia-skill-finder/SKILL.md`) -- **MEDIUM** SECURITY/Unknown (SQP-1): L032 instructs activation on any "how do I do X" request where X "might be a common task that an NVIDIA skill can help w (`SKILL.md:32`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: The `npx skills add nvidia/skills --list` command is invoked without a pinned package version. When npx resolves an unpi (`SKILL.md:74`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: Same unpinned `npx skills` invocation as above, used as the primary catalog-check mechanism. Without a version pin, any (`SKILL.md:103`) -- **MEDIUM** SECURITY/Unknown (RP1): MCP Rug Pull: Another instance of the same unpinned `npx skills` pattern. The cumulative effect of multiple unpinned invocations acros (`SKILL.md:113`) -- 14 additional finding(s) are available in the full evaluation artifacts. +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:74`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:103`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:183`) +- **MEDIUM** SECURITY/MCP Rug Pull (RP1): MCP Rug Pull: npx commands without a version suffix (e.g. @1.0.0) create a rug-pull risk if the upstream server is compromised and pub (`SKILL.md:113`) +- 10 additional finding(s) are available in the full evaluation artifacts. diff --git a/skills/nvidia-skill-finder/SKILL.md b/skills/nvidia-skill-finder/SKILL.md index e9aa2f887..3083c8e59 100644 --- a/skills/nvidia-skill-finder/SKILL.md +++ b/skills/nvidia-skill-finder/SKILL.md @@ -113,7 +113,13 @@ Catalog check means one of: npx skills add nvidia/skills --list ``` -Use the fallback catalog sources only if the CLI is unavailable, blocked, or fails: +- If running in Hermes Agent, do not run `npx`. Follow + [references/hermes-catalog.md](references/hermes-catalog.md) instead. +- Otherwise do not run `hermes` commands. The binary may exist on the machine + even when you are a different agent, and installing through it writes to + Hermes's skills directory, where you cannot load the result. + +Use the fallback catalog sources only if the lookup above is unavailable, blocked, or fails: - https://github.com/NVIDIA/skills/tree/main/skills - https://build.nvidia.com/skills - https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json @@ -149,6 +155,11 @@ If the agent target is unknown, omit `--agent` and let the CLI prompt: npx skills add nvidia/skills --skill --global --yes ``` +- If running in Hermes Agent, install with `hermes skills install` instead so + the skill is security scanned and lands in the active profile. Casing and + identifier rules are in + [references/hermes-catalog.md](references/hermes-catalog.md). + After install, tell the user to restart or reload the agent if their client does not pick up newly installed skills immediately. @@ -163,8 +174,8 @@ does not pick up newly installed skills immediately. Do not fabricate catalog entries or guess at skill names. If catalog lookup fails, say the catalog could not be checked and do not name a concrete slug or -emit an `npx skills add ... --skill ` install command. Offer to share the -catalog URL, retry lookup, or continue with general help. +emit an install command for it, in either form. Offer to share the catalog URL, +retry lookup, or continue with general help. ## When No Skills are Found diff --git a/skills/nvidia-skill-finder/evals/evals.json b/skills/nvidia-skill-finder/evals/evals.json index 7c351fc94..21fa07091 100644 --- a/skills/nvidia-skill-finder/evals/evals.json +++ b/skills/nvidia-skill-finder/evals/evals.json @@ -7,7 +7,7 @@ "ground_truth": "The agent identifies Decision Optimization as a strong NVIDIA taxonomy match, checks the live NVIDIA skills catalog, and recommends a current cuOpt routing skill rather than implementing a routing solver from memory.", "expected_behavior": [ "Treats vehicle routing as a strong NVIDIA Decision Optimization signal", - "Checks the live NVIDIA catalog (e.g. `npx skills add nvidia/skills --list`) before naming a specific skill", + "Checks the live NVIDIA catalog before naming a specific skill, using the lookup for the current agent (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`)", "Recommends a cuOpt routing-related skill if present in the catalog", "Asks before installing the skill", "Does not implement a routing solver before offering the skill" @@ -99,7 +99,7 @@ "ground_truth": "The agent treats Jetson driver installation as a strong NVIDIA Robotics / Jetson workflow signal, checks the live NVIDIA catalog before naming a skill, and recommends the current Jetson setup or quick-start skill if available.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Jetson driver / JetPack / BSP request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends a Jetson setup skill (e.g. jetson-quick-start) if present", "Explains that Jetson drivers are normally bundled with JetPack / Jetson Linux rather than installed as standalone desktop drivers", @@ -114,7 +114,7 @@ "ground_truth": "The agent treats Jetson setup as a strong NVIDIA Robotics / Jetson workflow signal, checks the live NVIDIA skills catalog before using product docs as the main answer, and recommends the current Jetson setup or quick-start skill if available.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Jetson setup request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends a Jetson setup skill (e.g. jetson-quick-start) if present", "Provides practical Jetson setup guidance only after the catalog lookup path", @@ -129,7 +129,7 @@ "ground_truth": "The agent treats Isaac Lab as a strong NVIDIA Robotics / Physical AI signal, checks the live NVIDIA skills catalog before giving general Isaac Lab guidance, and recommends a current Isaac-related skill only if the catalog has one.", "expected_behavior": [ "Invokes the NVIDIA skill finder for the Isaac Lab request", - "Attempts `npx skills add nvidia/skills --list` when shell access is available, or uses an approved catalog fallback if the CLI is unavailable", + "Attempts the catalog lookup for the current agent when shell access is available (`hermes skills search --source nvidia` in Hermes, otherwise `npx skills add nvidia/skills --list`), or uses an approved catalog fallback if it is unavailable", "Does not count developer.nvidia.com or docs.nvidia.com product documentation as the live catalog check", "Recommends an Isaac-related skill if present in the catalog, or says no strong catalog match was found without inventing a skill name", "Asks before installing the skill" diff --git a/skills/nvidia-skill-finder/references/hermes-catalog.md b/skills/nvidia-skill-finder/references/hermes-catalog.md new file mode 100644 index 000000000..30eb8026f --- /dev/null +++ b/skills/nvidia-skill-finder/references/hermes-catalog.md @@ -0,0 +1,76 @@ +# Hermes Agent: Catalog Lookup and Install + +Follow this instead of the `npx` commands in SKILL.md when you are running in +Hermes Agent. Install goes through Hermes so the skill is security scanned and +lands in the active profile, and `npx` stays unnecessary. + +`--source nvidia` requires Hermes 0.22 or newer; older versions reject the +flag. If it is rejected, drop `--source nvidia` and filter the results for +`NVIDIA/skills` identifiers yourself. + +## Catalog lookup + +List the whole catalog once and choose from it yourself: + +```bash +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ + | jq -r '.skills[] | "\(.name): \(.description[0:110])"' +``` + +If `jq` is unavailable, use this instead. Do not reach for `python3 -c`: Hermes +blocks inline interpreter scripts as "script execution via -e/-c flag", and the +nested quoting such a one-liner needs is a syntax error on Python 3.11 and +older. + +```bash +curl -sL https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json \ + | python3 -m json.tool | grep -E '"(name|description)":' | cut -c1-130 +``` + +Read the list and pick the best match on the descriptions. Prefer this over +searching. Searching looks cheaper but is not: it matches the query as a plain +case-insensitive substring over each entry's joined fields, so it misses +phrases (`jetson pinmux` returns nothing even though `jetson-customize-pinmux` +exists), and broad keywords return full descriptions for every hit. A handful +of searches costs more context than the complete listing above and can still +miss the right skill. + +Do not use `hermes skills browse` either. It costs more than the listing and +truncates both names and descriptions, so it is worse on accuracy and cost. + +## Confirm the identifier + +Once you have chosen a specific skill, look up that exact name to confirm it is +available in Hermes and to get its identifier: + +```bash +hermes skills search --source nvidia --json --limit 100 +``` + +An exact skill name is a narrow query, so this stays small. Pass `--limit 100` +anyway; the default is 25, which silently truncates broader matches. + +## Install + +Use the `Identifier` from that result, copied verbatim: + +```bash +hermes skills install +``` + +Casing determines trust, not the number of path segments. `NVIDIA/...` resolves +to the trusted first-party source; `nvidia/...` resolves to a community-trust +mirror with a placeholder description. Always use `NVIDIA`. + +The identifier may be a longer path such as `NVIDIA/skills/skills/`; +that form and the short `NVIDIA/skills/` both resolve to the same +trusted skill with the same scan hash. + +Confirm an identifier you are unsure about before installing it: + +```bash +hermes skills inspect +``` + +Hermes prompts for confirmation itself, so do not pass `--yes` on the user's +behalf. diff --git a/skills/nvidia-skill-finder/skill-card.md b/skills/nvidia-skill-finder/skill-card.md index b5487a7ed..99e251036 100644 --- a/skills/nvidia-skill-finder/skill-card.md +++ b/skills/nvidia-skill-finder/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Helps users discover, install, and start using NVIDIA agent skills by searching the live NVIDIA skills catalog and recommending matches for their task.
+Use for NVIDIA-related requests where an NVIDIA skill might help, even if the user did not ask for a skill; it triggers on NVIDIA products, hardware, software, SDKs, GPUs, Jetson/JetPack/L4T/BSP/SDK Manager/driver/flashing/setup, CUDA, NIM, NeMo, Omniverse/OpenUSD/SimReady, Isaac, RAPIDS/cuDF, cuPyNumeric, cuOpt, Dynamo, Holoscan, TensorRT, DeepStream, VSS, TAO, and NGC/NVCF, and does not apply to generic non-NVIDIA route, optimize, deploy, AI, video, data, or infrastructure tasks.
This skill is ready for commercial/non-commercial use.
@@ -7,9 +7,9 @@ This skill is ready for commercial/non-commercial use.
NVIDIA
### License/Terms of Use:
-CC-BY-4.0 AND Apache-2.0
+CC-BY-4.0 AND Apache 2.0
## Use Case:
-Developers and engineers use this skill to find and install NVIDIA-verified agent skills that match their task, covering domains such as Physical AI, Robotics, Vision AI, Data Science, Inference, Training, Decision Optimization, and GPU Development.
+Developers and engineers using AI coding agents (such as Claude Code, Codex, or Hermes Agent) use this skill to discover, recommend, and, with explicit user approval, install NVIDIA skills from the live NVIDIA skills catalog that match an NVIDIA product, task, or workflow.
### Deployment Geography for Use:
Global
@@ -25,58 +25,93 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
+- [Hermes Agent: Catalog Lookup and Install](references/hermes-catalog.md)
- [NVIDIA Taxonomy Routing](references/taxonomy-routing.md)
-- [NVIDIA Skills Catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
-- [NVIDIA Skills Catalog (Build Portal)](https://build.nvidia.com/skills)
-- [NVIDIA Agent Skills Documentation](https://docs.nvidia.com/skills)
+- [NVIDIA Skills catalog (GitHub)](https://github.com/NVIDIA/skills/tree/main/skills)
+- [NVIDIA Skills on build.nvidia.com](https://build.nvidia.com/skills)
+- [NVIDIA Skills catalog index (skills.sh.json)](https://raw.githubusercontent.com/NVIDIA/skills/main/skills.sh.json)
+- [NVIDIA Skills marketplace metadata](https://raw.githubusercontent.com/NVIDIA/skills/main/.github/scripts/marketplace/metadata.json)
## Skill Output:
-**Output Type(s):** [Analysis, Shell commands, Configuration instructions]
+**Output Type(s):** [Skill recommendations, Shell commands]
**Output Format:** [Markdown with inline bash code blocks]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [None]
+**Other Properties Related to Output:** [Recommends at most three skills ordered by confidence; install commands are run only after explicit user approval.]
## Evaluation Agents Used:
-- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
-- Codex (`openai/openai/gpt-5.5`)
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
## Evaluation Tasks:
-18 evaluation tasks (13 positive, 5 negative), each run with 3 attempts per task in isolated k8s-sandbox pods.
+18 evaluation tasks (13 positive, 5 negative); 1 attempt per task, each run in its own isolated sandbox pod (`k8s-sandbox` environment); each task was also attempted without the skill as a baseline. Evaluator version `1.6.9`; dataset digest `sha256:75b2b97f4c0057ea5bbadcfcd92d5561919ce56e7f03a93f629cb8bdd8f01281` (skill-evaluator-dataset-snapshot/1).
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
-- Correctness: Checks final-answer correctness against the reference answer.
-- Discoverability: Checks whether the expected skill was selected and the workflow executed.
-- Effectiveness: Checks whether the user's goal was achieved and the expected workflow behavior was followed.
-- Efficiency: Checks tool-call productivity and token usage efficiency.
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
Underlying evaluation signals used in this run:
-- `security`: Unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
-- `accuracy`: Final-answer correctness against the reference answer.
-- `goal_accuracy`: Whether the user's goal was achieved.
-- `behavior_check`: Whether the expected workflow behavior was followed.
-- `skill_efficiency`: Tool-call productivity (routing scored under Discoverability).
-- `token_efficiency`: Actual uncached prompt plus completion token usage.
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
## Evaluation Results:
-| Dimension | Tasks | Claude Code | Codex | -|---|---:|---:|---:| -| Overall | 18 | 92.0% | 89.0% | -| Security | 18 | 97.2% | 94.4% | -| Correctness | 18 | 96.7% | 100.0% | -| Discoverability | 18 | 100.0% | 94.6% | -| Effectiveness | 18 | 87.9% | 89.7% | -| Efficiency | 18 | 78.4% | 66.5% | +> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 94.2% — baseline ran, but no comparable score was available; uplift unavailable | 89.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 97.2% → 97.2% (±0.0 points) | 100.0% → 91.7% (-8.3 points) | +| Correctness | 64.4% → 96.7% (+32.3 points) | 74.4% → 96.7% (+22.3 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 36.4% → 95.8% (+59.4 points) | 58.6% → 98.6% (+40.0 points) | +| Efficiency | 81.3% — baseline ran, but no comparable score was available; uplift unavailable | 64.4% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.2% → 96.4% (-1.8 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 90.3% → 88.9% (-1.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 15 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 18 task(s) | ## Skill Version(s):
-0.3.0 (source: changelog, released 2025-05-21)
+0.3.0 (source: changelog, released 2025-05-21; git commit cc8c779)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/nvidia-skill-finder/skill.oms.sig b/skills/nvidia-skill-finder/skill.oms.sig index cdde023c3..4a6e12ff3 100644 --- a/skills/nvidia-skill-finder/skill.oms.sig +++ b/skills/nvidia-skill-finder/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICIwNTM4NDcwMjkyNDZlYjI1NTA2YzMyZjk0MDQ5NDJlOGMyNzM2MzdiYjA1ODI5MzhlMjAwYTU5ZGQ5Mzg0YmE1IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiZWJjN2RlMTc5OGIyYmJhZTE1ZjExOGMyNWVjZjQwOThkYThlN2Q4MmVhYjJhMjhjZWEyNTgwNTA1ZjQwZTBjOSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJmNTllNWMyYzllODljZDM1NjM3NjQyMjUyODBmOWE4ZjA1ZWExMTY0ZWZhOTc4OGZlYTU4OTJmNjViMDRkZjJjIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFnZW50cy9vcGVuYWkueWFtbCIsCiAgICAgICAgImRpZ2VzdCI6ICJmMmIyOTQyMjUyMDYxMjUwNTY0M2M2YTFkZDJmNDQxYjRhMWJmMzUwZmE1MWM2ZmU2ZWY0NWE1MjZhNzJiNzM5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiNzE2ZmNiM2ZiMTFkMDM2MjYyYTY5NjY0OTI4MmZlYTc2ZmE2MGQyNDIzYTc4MDFlMWQ0YTNlOGY3OTMzZjBlNCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3RheG9ub215LXJvdXRpbmcubWQiLAogICAgICAgICJkaWdlc3QiOiAiYTI0YjU5NGIwNmZhNGUxNmM4OTM3ZGE1NGI5YmZjMmNiZWNlOGJjYzY2YjlkMDJhNjcyOTQzNDAyMWExNjg0OCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjI5OWRkZjA3NzUzOTU3ODY2MzQxMzQ5YjQ5N2MxODQ4MWNmYzI1NWE5MTMxZWYzNTAyNDM4Y2Y3NDA3MTg3OTAiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMGGg1+VbtPZ6EXlNDc0xHBMjfZN0EAGw1oDeoYkA/BuRWHpci0EFllec/wl0PyaTswIxAJLsjn6E8DOrgQvHZgdt+8ILmYrhydTiQ6wLzhXUeJKe3vGsgSFDv0BLA1hZ1kp1ig==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibnZpZGlhLXNraWxsLWZpbmRlciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI5Nzg0OTEyM2QwMjY5NjI1MTZhYTlhODJhYzhhYzNiZTgxODM3ZTNiOTM5ODhhMzRlNjZjMDM2NWQ1ZTcxNzIxIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICJmZmRiM2YwMTUwZTQ0MDQzZGM5ODU1OWM1NzE1NzU3YzdhNmMyZTM1YzJhZGI4YzlkMzY4NTI5ZjQzZjM3MDNmIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjQyZDM2NDE2NDNjMTQyNzkyOGJhYmNjNzI2ZTA0OTExMjlhMTQwMjljZGJhZmJiYWJlYmNkMjdkMzIxMjdjMmQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiYWdlbnRzL29wZW5haS55YW1sIiwKICAgICAgICAiZGlnZXN0IjogImYyYjI5NDIyNTIwNjEyNTA1NjQzYzZhMWRkMmY0NDFiNGExYmYzNTBmYTUxYzZmZTZlZjQ1YTUyNmE3MmI3MzkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICIzZTQwYjc0ZjM3MWZkMzAzMDIzODI3MDE0MWE2N2FiMmM2OGEzNTJjODNmYzcxNmRkOTQ0MjcwODA3M2M0NzVhIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvaGVybWVzLWNhdGFsb2cubWQiLAogICAgICAgICJkaWdlc3QiOiAiNzZmYTAxYWQwY2JjZDJjODA4Y2FjOTlkM2Y1NjE1MDRlNjc5MDI3MzY2NTk1MDAwNDk0Yzk1NDA2YWI4NDM0MyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3RheG9ub215LXJvdXRpbmcubWQiLAogICAgICAgICJkaWdlc3QiOiAiYTI0YjU5NGIwNmZhNGUxNmM4OTM3ZGE1NGI5YmZjMmNiZWNlOGJjYzY2YjlkMDJhNjcyOTQzNDAyMWExNjg0OCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogImM4Yzc1NGI2YTZlNDlhMzEwZTRjMDM2YjljNjc0MmM2MmYzNWIxM2YxNzlhNjhkMzE0ODM5NzdmZjNjY2U0ZWYiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMAqQa0apK17qXOL2hm+jiqAoKkVbKLViZXBYTbY/J7wenFy9ZKIlW4BEqiTSdBKvIwIwRNpMQ7amt07GRT7cQZrScF7A1lzem+PTyJJkFUIHH2u0SRM0wyY+cFPmaYUkfaYL","keyid":""}]}} \ No newline at end of file diff --git a/versions.json b/versions.json index 3f6809531..49eae5547 100644 --- a/versions.json +++ b/versions.json @@ -2118,9 +2118,9 @@ { "name": "nvidia-skill-finder", "path": "skills/nvidia-skill-finder", - "content_digest": "sha256:da43951c0c8c69da5622f5299055228c70bc6345273e7869f3c59acb92af8310", - "last_commit": "13a0ce1d97a350e2a7c6d0ca82fea37d293af625", - "last_modified": "2026-10-01" + "content_digest": "sha256:63a07ba8ba0d12c9cb85c8a5dc4dabd52ec4fc56c7cb933a340aee85cb5724a2", + "last_commit": "5120ecb61e8bb246a16b3eab10366e4a1390b44b", + "last_modified": "2026-11-10" }, { "name": "nvshmem-collect-performance-data",