From 1721c44bfaf856ee3480a7a39a116d1fa56386ee Mon Sep 17 00:00:00 2001 From: mosheabr <257371078+mosheabr@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:49:28 +0000 Subject: [PATCH] chore: sync skills (Newton) --- README.md | 2 + skills/newton-performance/BENCHMARK.md | 169 +++++++++++++++++++++ skills/newton-performance/SKILL.md | 116 ++++++++++++++ skills/newton-performance/evals/EVAL.md | 35 +++++ skills/newton-performance/evals/evals.json | 154 +++++++++++++++++++ skills/newton-performance/skill-card.md | 118 ++++++++++++++ skills/newton-performance/skill.oms.sig | 1 + skills/newton-usage/BENCHMARK.md | 155 +++++++++++++++++++ skills/newton-usage/SKILL.md | 37 +++++ skills/newton-usage/evals/EVAL.md | 29 ++++ skills/newton-usage/evals/evals.json | 72 +++++++++ skills/newton-usage/skill-card.md | 84 ++++++++++ skills/newton-usage/skill.oms.sig | 1 + 13 files changed, 973 insertions(+) create mode 100644 skills/newton-performance/BENCHMARK.md create mode 100644 skills/newton-performance/SKILL.md create mode 100644 skills/newton-performance/evals/EVAL.md create mode 100644 skills/newton-performance/evals/evals.json create mode 100644 skills/newton-performance/skill-card.md create mode 100644 skills/newton-performance/skill.oms.sig create mode 100644 skills/newton-usage/BENCHMARK.md create mode 100644 skills/newton-usage/SKILL.md create mode 100644 skills/newton-usage/evals/EVAL.md create mode 100644 skills/newton-usage/evals/evals.json create mode 100644 skills/newton-usage/skill-card.md create mode 100644 skills/newton-usage/skill.oms.sig diff --git a/README.md b/README.md index 66c441ee..973c820e 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,7 @@ For non-interactive installs, global installs, agent-specific installs, updates, | **Nemotron** | Author end-to-end model development, customization, evaluation, and deployment pipelines using the NVIDIA AI stack. | [`nemotron-customize`](skills/nemotron-customize), [`nemotron-retrieval-recipes`](skills/nemotron-retrieval-recipes), [`nemotron-policy-generator`](skills/nemotron-policy-generator) | | **Nemotron Speech** | Deploy and operate NVIDIA Nemotron Speech (Riva) NIMs — ASR, TTS, and NMT, cloud-hosted via build.nvidia.com or self-hosted on your own GPU. | [`nemotron-speech`](skills/nemotron-speech), [`nemotron-asr-finetune`](skills/nemotron-asr-finetune) | | **Nemotron Voice Agent** | Create or refine NVIDIA voice agents (Cascaded or Omni) with Pipecat or LiveKit — speech customization, model selection, cloud or local deployment, and iteration. | [`nemotron-voice-agent-builder`](skills/nemotron-voice-agent-builder) | +| **Newton** | Agent skills for setting up, debugging, and optimizing Newton physics simulations using public documentation and packaged examples. | [`newton-usage`](skills/newton-usage), [`newton-performance`](skills/newton-performance) | | **Nsight** | Agent-facing assets for Nsight developer workflows, including skills that help agents work with NVIDIA developer tools. | [`compute-sanitizer-diagnose-cuda-bugs`](skills/compute-sanitizer-diagnose-cuda-bugs) | | **NVIDIA App** | NVIDIA App MCP: drivers, games, laptops, overlay. Check drivers, manage and optimize games, configure laptop features. | [`nvidia-app`](skills/nvidia-app) | | **NVIDIA Broadcast** | Control NVIDIA Broadcast through its MCP gateway: apply audio and video effects, process local media, select devices, and change camera resolution. | [`nvidia-broadcast`](skills/nvidia-broadcast) | @@ -241,6 +242,7 @@ Per-product source repo links: | **Nemotron** | [Issues](https://github.com/NVIDIA-NeMo/Nemotron/issues) | [Discussions](https://github.com/NVIDIA-NeMo/Nemotron/discussions) | [Contributing](https://github.com/NVIDIA-NeMo/Nemotron/blob/main/CONTRIBUTING.md) | [Security](https://github.com/NVIDIA-NeMo/Nemotron/blob/main/SECURITY.md) | | **Nemotron Speech** | [Issues](https://github.com/nvidia-riva/Nemotron-speech-skills/issues) | — | [Contributing](https://github.com/nvidia-riva/Nemotron-speech-skills/blob/main/CONTRIBUTING.md) | [Security](https://github.com/nvidia-riva/Nemotron-speech-skills/blob/main/SECURITY.md) | | **Nemotron Voice Agent** | [Issues](https://github.com/NVIDIA-AI-Blueprints/nemotron-voice-agent/issues) | — | [Contributing](https://github.com/NVIDIA-AI-Blueprints/nemotron-voice-agent/blob/main/CONTRIBUTING.md) | [Security](https://github.com/NVIDIA-AI-Blueprints/nemotron-voice-agent/blob/main/SECURITY.md) | +| **Newton** | [Issues](https://github.com/NVIDIA/newton-skills/issues) | — | [Contributing](https://github.com/NVIDIA/newton-skills/blob/main/CONTRIBUTING.md) | [Security](https://github.com/NVIDIA/newton-skills/blob/main/SECURITY.md) | | **Nsight** | [Issues](https://github.com/NVIDIA/nsight/issues) | — | [Contributing](https://github.com/NVIDIA/nsight/blob/main/CONTRIBUTING.md) | [Security](https://github.com/NVIDIA/nsight/blob/main/SECURITY.md) | | **NVIDIA App** | [Issues](https://github.com/NVIDIA/nvidia-app-skills/issues) | — | [Contributing](https://github.com/NVIDIA/nvidia-app-skills/blob/main/CONTRIBUTING.md) | [Security](https://github.com/NVIDIA/nvidia-app-skills/blob/main/SECURITY.md) | | **NVIDIA Broadcast** | [Issues](https://github.com/NVIDIA/Broadcast-Skills/issues) | [Discussions](https://github.com/NVIDIA/Broadcast-Skills/discussions) | [Contributing](https://github.com/NVIDIA/Broadcast-Skills/blob/main/CONTRIBUTING.md) | [Security](https://github.com/NVIDIA/Broadcast-Skills/blob/main/SECURITY.md) | diff --git a/skills/newton-performance/BENCHMARK.md b/skills/newton-performance/BENCHMARK.md new file mode 100644 index 00000000..944aa301 --- /dev/null +++ b/skills/newton-performance/BENCHMARK.md @@ -0,0 +1,169 @@ +# Skill Benchmark: newton-performance + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `newton-performance` +- Validation status: `passed` +- Evaluation date: 2026-11-11 +- Evaluator version: `1.6.9` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 12 evaluation tasks (11 positive, 1 negative) +- Dataset digest: `sha256:6ecce6eb8f7b83630e185404459abc12d2a3d0ce0ab5c386ad9dbe1f77ef09f9` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| newton-performance | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| newton-performance | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| newton-performance | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| newton-performance | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | +| newton-performance | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.4` | nondeterministic / provider_defaults | + +- Controls for `newton-performance`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 87.3% — baseline ran, but no comparable score was available; uplift unavailable | 84.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 88.3% → 91.7% (+3.4 points) | 93.3% → 98.3% (+5.0 points) | +| Discoverability | 65.5% — baseline ran, but no comparable score was available; uplift unavailable | 74.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 87.5% → 83.8% (-3.7 points) | 94.2% → 95.8% (+1.6 points) | +| Efficiency | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | 54.9% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 97.6% (-0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 88.1% → 85.5% (-2.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 1,256,391 | 1,016,996 | +239,395 | +23.54% | skill 12/12; base 12/12 | +| claude-code | newton-performance-blocker | 187,700 | 144,854 | +42,846 | +29.58% | skill 1/1; base 1/1 | +| claude-code | newton-performance-context-many-worlds | 150,891 | 110,428 | +40,463 | +36.64% | skill 1/1; base 1/1 | +| claude-code | newton-performance-coupled-sweep | 113,462 | 110,372 | +3,090 | +2.80% | skill 1/1; base 1/1 | +| claude-code | newton-performance-explicit-review | 76,185 | 37,891 | +38,294 | +101.06% | skill 1/1; base 1/1 | +| claude-code | newton-performance-implicit-slow-motion | 37,150 | 36,931 | +219 | +0.59% | skill 1/1; base 1/1 | +| claude-code | newton-performance-implicit-startup | 74,826 | 37,608 | +37,218 | +98.96% | skill 1/1; base 1/1 | +| claude-code | newton-performance-multiturn-into-scope | 149,285 | 72,355 | +76,930 | +106.32% | skill 1/1; base 1/1 | +| claude-code | newton-performance-multiturn-out-of-scope | 36,220 | 107,889 | -71,669 | -66.43% | skill 1/1; base 1/1 | +| claude-code | newton-performance-negative-correctness | 109,246 | 109,220 | +26 | +0.02% | skill 1/1; base 1/1 | +| claude-code | newton-performance-negative-solver-api | 142,752 | 71,261 | +71,491 | +100.32% | skill 1/1; base 1/1 | +| claude-code | newton-performance-negative-usage | 106,087 | 105,972 | +115 | +0.11% | skill 1/1; base 1/1 | +| claude-code | newton-performance-negative-warp-kernel | 72,587 | 72,215 | +372 | +0.52% | skill 1/1; base 1/1 | +| codex | All cases | 663,094 | 413,055 | +250,039 | +60.53% | skill 12/12; base 12/12 | +| codex | newton-performance-blocker | 89,357 | 58,119 | +31,238 | +53.75% | skill 1/1; base 1/1 | +| codex | newton-performance-context-many-worlds | 149,774 | 56,632 | +93,142 | +164.47% | skill 1/1; base 1/1 | +| codex | newton-performance-coupled-sweep | 68,463 | 58,925 | +9,538 | +16.19% | skill 1/1; base 1/1 | +| codex | newton-performance-explicit-review | 59,220 | 27,088 | +32,132 | +118.62% | skill 1/1; base 1/1 | +| codex | newton-performance-implicit-slow-motion | 45,124 | 30,648 | +14,476 | +47.23% | skill 1/1; base 1/1 | +| codex | newton-performance-implicit-startup | 45,450 | 13,816 | +31,634 | +228.97% | skill 1/1; base 1/1 | +| codex | newton-performance-multiturn-into-scope | 66,054 | 26,310 | +39,744 | +151.06% | skill 1/1; base 1/1 | +| codex | newton-performance-multiturn-out-of-scope | 21,308 | 18,111 | +3,197 | +17.65% | skill 1/1; base 1/1 | +| codex | newton-performance-negative-correctness | 39,037 | 53,005 | -13,968 | -26.35% | skill 1/1; base 1/1 | +| codex | newton-performance-negative-solver-api | 30,522 | 30,295 | +227 | +0.75% | skill 1/1; base 1/1 | +| codex | newton-performance-negative-usage | 17,537 | 17,478 | +59 | +0.34% | skill 1/1; base 1/1 | +| codex | newton-performance-negative-warp-kernel | 31,248 | 22,628 | +8,620 | +38.09% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 1,919,485 | 1,430,051 | +489,434 | +34.22% | skill 24/24; base 24/24 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 12 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 12 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/newton-performance/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/newton-performance/SKILL.md`) +- **MEDIUM** QUALITY/quality_discoverability: Description contains vague words (`skills/newton-performance/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/newton-performance/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/newton-performance/SKILL.md`) +- 7 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/newton-performance/SKILL.md b/skills/newton-performance/SKILL.md new file mode 100644 index 00000000..89c227dd --- /dev/null +++ b/skills/newton-performance/SKILL.md @@ -0,0 +1,116 @@ +--- +name: newton-performance +description: Measure or improve the runtime, startup time, memory use, or throughput of a Newton simulation that already behaves correctly, or review a Newton step loop for performance problems such as CUDA graph capture mistakes and host synchronization. Not for setting up simulations, general solver or API questions, debugging unstable or incorrect physics without a performance goal, or optimizing Warp code that does not use Newton. +--- + +# Newton Performance + +Make a correct Newton workload faster without changing its results. Newton's +documentation holds the technical guidance; this skill orders the work and +defines what counts as evidence. + +## 1. Scope the request + +- If the simulation is unstable, produces invalid states, or the user's goal is + correctness rather than speed, use `newton-usage` instead and stop here. +- Decide the mode. A **plan** request (how should I profile or optimize this?) + gets a documentation route and one bounded experiment. A **review** request + (is this loop correct and fast?) gets findings against the documentation. + In both, reading code and documentation is fine, but do not run the + simulation, benchmarks, or profilers unless asked. A **measure** or + **implement** request runs the workflow below. +- Record the workload, the objective, and the timing scope. Keep these scopes + separate, because they have different causes: startup (import, model build, + kernel compilation), warm stepping, rendering, host/device transfers, memory, + and training throughput. +- Record the fidelity checks that must still pass after a change, for example + finite state, no contact overflow, and a trajectory or force metric against a + reference run. + +## 2. Gather context once + +Reuse facts the user already gave. Otherwise run once: + +``` +python -c "import sys, newton; from importlib.metadata import version; print(sys.executable, version('newton'), version('warp-lang'), newton.__file__)" +``` + +Check GPU availability only when the experiment depends on it. If the workload, +a required profiler, or required permissions are missing, report exactly what +is missing and stop. If elevated privileges are needed (for example GPU +performance counters), give the exact command for the user to run and stop. + +## 3. Route to the documentation + +Use the Newton documentation for the installed release, +`https://newton-physics.github.io/newton//`, and flag a mismatch +before relying on an API or default. Open only the page the task needs; paths +are relative to that root: + +- `concepts/simulation_tuning.html`: trading substeps, iterations, and contact + settings against accuracy. +- `concepts/collisions.html`: broad phase, geometry, contact capacity, and + collision frequency. +- `solvers/index.html` and the API reference: solver options, capacities, + `ModelFlags`, and `State.assign` for odd substep counts in a captured frame. +- Warp's Profiling page for instrumentation and timing, under the installed + Warp version's documentation, `https://nvidia.github.io/warp/v./`. + The page path differs between Warp versions, so find it from that version's + index. If that version is not published, use `stable` and flag the mismatch. + +Cite the page and section each recommendation relies on. + +When the user only needs a documentation location, give the URL without +opening it. + +## 4. Measure, change, verify + +1. Measure a reproducible baseline for the recorded workload and timing scope. +2. Choose the profiling path for that scope: startup (import, model build, + compilation), eager stepping, or captured-graph replay. Each needs different + tools and timing boundaries; Warp's profiling guide gives the tools. + Exclude warm-up, compilation, and the first graph launch from steady-state + timing, synchronize at the timing boundaries, and add only the + instrumentation the question needs. +3. Collect the profile and inspect it: identify the dominant kernels, gaps, or + synchronization points, and state the evidence. Do not choose a change + before this step, and do not start from universal tuning values. +4. Select a documented lever whose accuracy tradeoff fits the task. Change one + factor at a time. When settings interact (for example collision + frequency, contact model, mesh resolution, and gap), run a small bounded + sweep over the coupled set instead. +5. Repeat before and after measurements under the same conditions and + instrumentation to estimate variance, and re-run the fidelity checks from + section 1. +6. Keep a change only if it meets the objective and passes the fidelity + checks; otherwise revert it. + +## 5. Review checks + +When reviewing a Newton step loop, check these against the documentation +selected in section 3 and cite the page: + +- the whole frame, including collision, is captured in one graph and warmed up + before capture; +- an odd substep count with a plain state swap inside a captured frame; +- arrays replaced instead of written in place after capture, or Python values + expected to change between replays; +- host reads (`.numpy()`, contact counts, printing) inside the frame; +- capacities sized from defaults rather than measured peaks, and overflow never + checked; +- rendering or logging inside the substep loop, or included in simulation + timings. + +Credit any correct fix the documentation supports, not only the first one +listed there. + +## 6. Report + +Report the Newton and Warp versions, the environment, the documentation used, +the workload and timing scope, the change, the before and after measurements +with variance, the fidelity evidence, and the largest remaining cost or +blocker. Label proposed experiments separately from measurements actually +collected. CPU measurements or source inspection do not establish a GPU +speedup. + +Compatibility with specific Newton releases has not been validated. diff --git a/skills/newton-performance/evals/EVAL.md b/skills/newton-performance/evals/EVAL.md new file mode 100644 index 00000000..4b4e8a10 --- /dev/null +++ b/skills/newton-performance/evals/EVAL.md @@ -0,0 +1,35 @@ +# Evaluation Scope + +This smoke suite measures routing and the measurement workflow of the +performance skill for the Newton wheel. It covers explicit, implicit, +contextual, multi-turn, and negative prompts. Tasks request reviews, plans, or +instructions only; they do not require a GPU or execute simulations. These +cases do not establish measured speedups or a supported Newton version range. + +## Behaviors + +- Separate timing scopes such as startup, warm stepping, and rendering, and + keep fidelity checks alongside every measurement. +- Use bounded coupled sweeps when settings interact. +- Report missing workloads or prerequisites, and label proposed experiments + separately from collected measurements. +- Find graph-capture and host-synchronization problems in a step loop and give + a correct fix; any fix supported by the Newton documentation receives credit. +- Size collision and solver buffers with the correct per-world or total scope. +- Leave setup, solver and API questions, correctness debugging, and Warp-only + kernel work to other skills, including after a conversation moves out of + performance work. + +## Notes + +Run paired trials with and without the skill in otherwise identical environments +on both Claude Code and Codex. Use the same public documentation access in both +arms. Repeat trials to measure variance. Report task correctness, routing, +total token consumption including skill loading and retries, and elapsed time. +Assertions judge the task outcome; routing is measured separately through +`expected_skill`, and the baseline is judged on the same outcome assertions. + +The multi-turn cases encode the earlier turns in the prompt. The startup and +coupled-sweep cases are adapted from the `newton-optimize-simulation` proposal. +No live trial results exist yet for this version of the skill. Keep larger +executable performance evaluations separate from this routing suite. diff --git a/skills/newton-performance/evals/evals.json b/skills/newton-performance/evals/evals.json new file mode 100644 index 00000000..6aa46ced --- /dev/null +++ b/skills/newton-performance/evals/evals.json @@ -0,0 +1,154 @@ +{ + "skill_name": "newton-performance", + "evals": [ + { + "id": "newton-performance-explicit-review", + "prompt": "Use newton-performance to review this Newton step loop for performance and correctness problems. It is captured with wp.ScopedCapture once and replayed every frame.\n\n```python\nself.sim_substeps = 5\n\ndef simulate(self):\n for _ in range(self.sim_substeps):\n self.state_0.clear_forces()\n self.pipeline.collide(self.state_0, self.contacts)\n self.solver.step(self.state_0, self.state_1, self.control, self.contacts, self.sim_dt)\n self.state_0, self.state_1 = self.state_1, self.state_0\n n = self.contacts.rigid_contact_count.numpy()[0]\n if n > self.max_seen:\n self.max_seen = n\n```\n\nGive the findings and a corrected loop; do not run anything.", + "expected_skill": "newton-performance", + "expected_script": null, + "expected_output": "Explains that with an odd substep count the Python swap runs only during capture, so each replay reads a buffer that holds an older state and the simulation advances fewer substeps per frame without an error. Gives a correct fix: copying the final state with State.assign, an even substep count, or alternating two graphs with a consistent readable state. Explains that the .numpy() contact-count read synchronizes with the host and cannot be captured, and that the Python max tracking would only run during capture; moves peak or overflow tracking to the device or to occasional checks outside the graph. Provides a corrected loop and does not claim to have run it.", + "assertions": [ + "After one replay of the corrected loop, the state that callers read has advanced by the intended frame duration and holds the correct output state, whether the fix keeps five substeps or adjusts the substep count and dt.", + "Identifies the per-substep .numpy() read as a host synchronization that is invalid or frozen inside a captured graph, and removes it from the frame.", + "Keeps peak or overflow contact tracking, either on the device inside the graph or as an occasional synchronized check outside it, rather than deleting it.", + "Keeps force clearing, collision, and the solver step inside the captured frame.", + "Does not claim the corrected loop was run or benchmarked." + ] + }, + { + "id": "newton-performance-implicit-slow-motion", + "prompt": "After I wrapped my Newton simulate() in a CUDA graph, objects fall noticeably slower than before, but nothing errors. Without the graph it looks right. I use 3 substeps per frame and SolverSemiImplicit. What is likely going on and how do I confirm it? Instructions only.", + "expected_skill": "newton-performance", + "expected_script": null, + "expected_output": "Attributes the slow motion to the odd substep count combined with a Python state swap inside the captured frame: the swap runs once during capture, so each replay starts from an older state. Proposes confirming it by comparing captured and eager runs with odd and even substep counts, and gives a correct fix such as State.assign on the last substep, an even substep count, or alternating graphs.", + "assertions": [ + "Attributes the slowdown to the odd substep count combined with the Python swap inside the captured graph.", + "Proposes a comparison between captured and eager execution that includes an odd substep count.", + "Gives a fix that makes each replay advance all substeps.", + "Does not claim the diagnosis was confirmed by running code." + ] + }, + { + "id": "newton-performance-context-many-worlds", + "prompt": "We scale a Newton robot scene from 64 to 8192 worlds with CollisionPipeline and SolverMuJoCo (use_mujoco_contacts=False). GPU memory explodes and each step got much slower than linear. We pass rigid_contact_max computed as our per-world peak plus 25%. What should we check and change? Do not change the environment.", + "expected_skill": "newton-performance", + "expected_script": null, + "expected_output": "Explains that CollisionPipeline capacities are totals across all worlds, so a per-world value drops contacts at high world counts, and that MuJoCo's contact and constraint capacities are per world and grow with the world count. Recommends measuring peak counts at the target world count, checking overflow outside the per-step loop, reviewing broad phase and collision geometry, and measuring throughput and memory at several world counts. Points to the Newton documentation for the installed release for exact parameter names and defaults.", + "assertions": [ + "States that the collision pipeline's rigid contact capacity covers all worlds, so a per-world value is too small.", + "Identifies per-world solver capacities multiplied by the world count as a memory cost.", + "Recommends sizing from measured peaks and checking overflow outside the per-step loop.", + "Does not claim measurements were taken." + ] + }, + { + "id": "newton-performance-implicit-startup", + "prompt": "The first run of my Newton simulation takes almost a minute before anything moves, but later frames are fast. How do I find out whether model building, kernel compilation, allocation, or stepping is responsible? Give me a plan; do not run anything.", + "expected_skill": "newton-performance", + "expected_script": null, + "expected_output": "Treats this as a startup-cost question. Defines separate timing regions for import, model building and finalize, kernel compilation and loading, allocation and graph capture, and warm stepping, with synchronization at each boundary. Notes that Warp reports module load and compile times and that the kernel cache affects repeated runs. Does not attribute the first-run cost to steady-state stepping and does not claim measurements.", + "assertions": [ + "Separates compilation, loading, and allocation from warm stepping in the measurement plan.", + "Synchronizes the device at the timing boundaries or otherwise accounts for asynchronous execution.", + "Accounts for kernel caching between runs.", + "Does not claim measurements were taken." + ] + }, + { + "id": "newton-performance-coupled-sweep", + "prompt": "A Newton peg-in-hole insertion workload is correct but too slow. Collision frequency, the contact model, mesh resolution, and the shape gap may interact. Define a bounded experiment that improves throughput without invalidating the insertion. Do not assume universal default values or claim a speedup without measurements from the target GPU.", + "expected_skill": "newton-performance", + "expected_script": null, + "expected_output": "Treats the four settings as coupled factors, fixes the workload and the acceptance criteria (insertion success, contact forces or penetration within bounds, no contact overflow), and proposes a small bounded sweep over a few levels of each factor with repeated measurements on the target GPU. Accepts a configuration only if it meets the throughput objective and the insertion checks. States that no speedup has been measured yet.", + "assertions": [ + "Treats the settings as interacting and proposes a bounded sweep rather than changing them independently.", + "Defines acceptance criteria for insertion validity before measuring.", + "Requires repeated measurements on the target GPU.", + "Does not propose universal default values or claim a speedup." + ] + }, + { + "id": "newton-performance-blocker", + "prompt": "Measure how much faster our Newton training environment runs with CUDA graph capture than without it, and report the speedup. The environment is in our team's cluster repository.", + "expected_skill": "newton-performance", + "expected_script": null, + "expected_output": "Looks for the workload once, finds that the environment code and a GPU target are not available in the current workspace, and reports exactly what is missing (the workload, and the GPU or access needed to run it). Describes the measurement it would run, labeled as a proposal, and does not report a speedup.", + "assertions": [ + "Reports which prerequisites are missing instead of inventing a workload.", + "Labels any described measurement as proposed, not collected.", + "Does not report a speedup number." + ] + }, + { + "id": "newton-performance-multiturn-into-scope", + "prompt": "Earlier in this conversation:\nUser: My Newton gripper keeps dropping the cube.\nAssistant: The friction coefficient on the fingers was too low; raising it fixed the grasp in your test.\n\nUser: Thanks, the grasp is reliable now. The problem is speed: with 4096 worlds each frame takes about 40 ms, and we need it under 10 ms. Where should we start? Plan only.", + "expected_skill": "newton-performance", + "expected_script": null, + "expected_output": "Recognizes that the request has moved from correctness to performance. Records the objective (frame time under 10 ms at 4096 worlds) and the fidelity check (grasp success must not regress). Plans a baseline measurement that separates collision, solver, and rendering, then checks graph capture, host synchronization, buffer capacities, and collision cost before trading substeps or iterations for speed.", + "assertions": [ + "Keeps the grasp success rate as a fidelity check for any change.", + "Plans a baseline measurement before proposing changes.", + "Considers overhead such as graph capture, host synchronization, or buffer sizing before reducing substeps or iterations.", + "Does not claim measurements were taken." + ] + }, + { + "id": "newton-performance-multiturn-out-of-scope", + "prompt": "Earlier in this conversation:\nUser: Help me capture my Newton step loop in a CUDA graph.\nAssistant: Done. The captured loop matches the eager loop and runs 4x faster.\n\nUser: Great. Different question: a box placed on the ground slowly sinks into it over a few seconds, both with and without the graph. Speed is fine now. What should I check?", + "expected_skill": "newton-usage", + "expected_script": null, + "expected_output": "Treats the new question as a correctness problem rather than performance work. Routes to collision, contact, and simulation-tuning documentation, asks for the solver and contact settings, and suggests checks such as contact stiffness, substeps, and collision geometry. Does not continue the performance workflow.", + "assertions": [ + "Treats the sinking as a contact or tuning problem, not a performance problem.", + "Does not propose performance changes or benchmarks.", + "Does not assert a root cause without evidence." + ] + }, + { + "id": "newton-performance-negative-usage", + "prompt": "I just installed the Newton physics wheel. How do I list and run one of the packaged examples? Give me the commands only.", + "expected_skill": "newton-usage", + "expected_script": null, + "expected_output": "Gives python -m newton.examples --list and python -m newton.examples . Does not discuss performance tuning, graph capture, or buffer sizing.", + "assertions": [ + "Provides a command that lists the packaged examples and a command that runs one.", + "Does not introduce performance-optimization advice." + ] + }, + { + "id": "newton-performance-negative-solver-api", + "prompt": "In Newton, what is the difference between SolverXPBD and SolverMuJoCo for articulated robots, and which one supports joint armature?", + "expected_skill": "newton-usage", + "expected_script": null, + "expected_output": "Answers from the Newton solver documentation and feature matrix for the installed release. Does not turn the question into a performance review.", + "assertions": [ + "Points to the solver documentation or feature support table.", + "Does not introduce graph capture, benchmarking, or buffer-sizing advice." + ] + }, + { + "id": "newton-performance-negative-correctness", + "prompt": "My Newton cloth simulation with SolverVBD explodes after a few seconds when the cloth touches a sphere. I do not care about speed yet. How do I find the cause?", + "expected_skill": "newton-usage", + "expected_script": null, + "expected_output": "Treats this as a stability problem. Routes to the VBD and simulation-tuning documentation, asks for the solver settings and a reproducer, and suggests checks such as the timestep, contact stiffness, and collision settings. Does not propose speed optimizations.", + "assertions": [ + "Treats the explosion as a stability or correctness problem.", + "Does not propose speed optimizations or benchmarks.", + "Does not assert a root cause without evidence." + ] + }, + { + "id": "newton-performance-negative-warp-kernel", + "prompt": "I wrote a Warp kernel for a 2D Eulerian fluid solver (no Newton involved). It is memory-bound. How can I improve its memory access pattern?", + "expected_skill": null, + "expected_script": null, + "expected_output": "Answers with Warp and CUDA memory-access guidance, such as coalesced access, data layout, and tiles. Does not load Newton skills or discuss Newton step loops.", + "assertions": [ + "Does not read newton-performance.", + "Gives Warp or CUDA memory-access guidance for the kernel.", + "Does not attribute the problem to Newton." + ] + } + ] +} diff --git a/skills/newton-performance/skill-card.md b/skills/newton-performance/skill-card.md new file mode 100644 index 00000000..a5a4ba9e --- /dev/null +++ b/skills/newton-performance/skill-card.md @@ -0,0 +1,118 @@ +## Description:
+Measure or improve the runtime, startup time, memory use, or throughput of a Newton simulation that already behaves correctly, or review a Newton step loop for performance problems such as CUDA graph capture mistakes and host synchronization.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+CC-BY-4.0 AND Apache 2.0
+## Use Case:
+Developers and engineers using released Newton physics simulator wheels use this skill to scope, profile, and optimize the performance of an already-correct Newton simulation, or to review a Newton step loop for CUDA graph capture and host synchronization problems, with recommendations routed to the version-matched Newton and Warp documentation.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [Newton physics simulator (GitHub)](https://github.com/newton-physics/newton)
+- [Newton installation guide](https://newton-physics.github.io/newton/latest/guide/installation.html)
+- [Newton documentation (version-matched root: newton-physics.github.io/newton//)](https://newton-physics.github.io/newton/latest/)
+- [Warp documentation (version-matched root: nvidia.github.io/warp/v./)](https://nvidia.github.io/warp/stable/)
+ + +## Skill Output:
+**Output Type(s):** [Analysis, Code, Shell commands]
+**Output Format:** [Markdown with inline Python and shell code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Reports Newton and Warp versions, documentation cited, timing scope, before/after measurements with variance, and fidelity evidence; proposed experiments are labeled separately from collected measurements.]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+12 evaluation tasks (11 positive, 1 negative), 1 attempt per task, run in the `k8s-sandbox` environment with each task attempt in its own isolated sandbox pod; each task was also attempted without the skill as a baseline. Dataset digest: `sha256:6ecce6eb8f7b83630e185404459abc12d2a3d0ce0ab5c386ad9dbe1f77ef09f9` (skill-evaluator-dataset-snapshot/1). Evaluator version `1.6.9`; evaluation date 2026-11-11.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from `security` (100%).
+- Correctness: Is the answer correct? Scored from `accuracy` (100%).
+- Discoverability: Was the right skill loaded when needed? Scored from `skill_execution` (100%).
+- Effectiveness: Did the skill help complete the task? Scored from `goal_accuracy` (50%) + `behavior_check` (50%).
+- Efficiency: Did it avoid wasted tool calls and token usage? Scored from `skill_efficiency` (50%) + `token_efficiency` (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Security: unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Skill Execution: whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool Productivity: tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy`: Accuracy: final-answer correctness against the reference answer.
+- `goal_accuracy`: Goal Accuracy: whether the user's goal was achieved.
+- `behavior_check`: Behavior Check: whether the expected workflow behavior was followed.
+- `token_efficiency`: Token Efficiency: actual uncached prompt plus completion usage (50% of Efficiency).
+ + + +## Evaluation Results:
+> ✅ **Overall verdict: PASS — Recommended for publication** + +### Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +### Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 87.3% — baseline ran, but no comparable score was available; uplift unavailable | 84.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 88.3% → 91.7% (+3.4 points) | 93.3% → 98.3% (+5.0 points) | +| Discoverability | 65.5% — baseline ran, but no comparable score was available; uplift unavailable | 74.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 87.5% → 83.8% (-3.7 points) | 94.2% → 95.8% (+1.6 points) | +| Efficiency | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | 54.9% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +#### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 98.0% → 97.6% (-0.4 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 88.1% → 85.5% (-2.6 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +### Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 12 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 12 task(s) | + +## Skill Version(s):
+de81080 (source: git SHA, committed 2026-11-04)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/newton-performance/skill.oms.sig b/skills/newton-performance/skill.oms.sig new file mode 100644 index 00000000..1a36096e --- /dev/null +++ b/skills/newton-performance/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibmV3dG9uLXBlcmZvcm1hbmNlIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjRhMGY3ZWM4Nzc5YmVhMmM0Y2U3YjBkYTY1ZDBkNWU5NDJjOGEwZDNiMjJkODYyN2U1OWE3MzIwZDU1ZTMxMTEiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjMzYWRjYjVkM2E1ZGMzMGJjNmY1OGJiZTQ0MTcxOGIzYTM3MjcxMzRjY2U4ZWVhNmYzMmIyMWQxMTMxZmEyMzYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiMWEyZTE0OWI4YzJjM2ViZTViNTAxOTI4OWNhMjJhZmZjYzllZmZkNWYwYjgwNzJhOTU1ZmE0ZDQ2ZmQzODVlMiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9FVkFMLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjgzZWU1Mjc4NzAyZWQzOGI0NmJlYTVlMzkxNTU3NTE4YzcyMGQwNzE0MWUwZTQzNThjNDEyMGM3Yzk4MzZhZWMiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICJkYTdhYzNkYmM3ZTMxMDNiMGI1MzkyYTA5YjE5YWZlZDBmMDkxMmQxMmY2M2JhYzVlODBmZDM0YjhiN2VlZDJkIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2Q5MzY5ZWQwNmFlMzZkNjgzODkxNTQyZWFkM2IxOWE2YTlhNDk3MWE1MWY1MTNmN2FmNmZmOGM1ZWIyNzI0OSIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDv1vIvsKjXbJ0IovvesKhBHrVDAW13gBazmyGDqNc3si85JyKh+c928mXWBPoojckCMCPWA+D7ya1t5T7W+eThXUpUp95lqANyfwfbQHsvA/vpjI4b+ReKEzopHWXTYiDpWg==","keyid":""}]}} \ No newline at end of file diff --git a/skills/newton-usage/BENCHMARK.md b/skills/newton-usage/BENCHMARK.md new file mode 100644 index 00000000..bf993c8c --- /dev/null +++ b/skills/newton-usage/BENCHMARK.md @@ -0,0 +1,155 @@ +# Skill Benchmark: newton-usage + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `newton-usage` +- Validation status: `passed` +- Evaluation date: 2026-10-30 +- Evaluator version: `1.6.4` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`), Codex (`openai/openai/gpt-6.1-sol`) +- Tasks: 5 evaluation tasks (3 positive, 2 negative) +- Dataset digest: `sha256:f186dac7d2c4e6f63962549401aecd2144324c46da0fee1334c49af5fc19b756` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## Tier 1 LLM Provenance + +SkillSpector provider, model, analyzer revision, and determinism evidence retained for this scan. + +| Skill | Provider | Adapter | Analyzer | Model | Revision | Determinism | +| --- | --- | --- | --- | --- | --- | --- | +| newton-usage | nv_inference | openai | mcp_tool_poisoning | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.2` | nondeterministic / provider_defaults | +| newton-usage | nv_inference | openai | semantic_developer_intent | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.2` | nondeterministic / provider_defaults | +| newton-usage | nv_inference | openai | semantic_quality_policy | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.2` | nondeterministic / provider_defaults | +| newton-usage | nv_inference | openai | semantic_security_discovery | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.2` | nondeterministic / provider_defaults | +| newton-usage | nv_inference | openai | meta_analyzer | `aws/anthropic/bedrock-claude-opus-5-5` | `2.13.2` | nondeterministic / provider_defaults | + +- Controls for `newton-usage`: temperature=not set (source=provider_default, forwarded=no); seed=not set (source=unset, forwarded=no); reasoning_effort=not set (source=provider_default, forwarded=no) + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 97.7% — baseline ran, but no comparable score was available; uplift unavailable | 89.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 80.0% → 92.0% (+12.0 points) | 80.0% → 100.0% (+20.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 66.0% → 97.5% (+31.5 points) | 58.0% → 87.5% (+29.5 points) | +| Efficiency | 99.2% — baseline ran, but no comparable score was available; uplift unavailable | 68.8% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +### Comparison details + +Shared-source comparisons use the same signals and weights on both conditions. They do not replace the full dimension scores above or change the publication verdict. + +| Agent | Dimension | Basis | Baseline → With skill (uplift) | Explanation | +| --- | --- | --- | --- | --- | +| claude-code | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| claude-code | Efficiency | Token-only comparison | 93.1% → 98.4% (+5.3 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | +| codex | Discoverability | Dimension comparison | N/A | Discoverability is not applicable to the no-skill baseline; there is no installed skill to discover. | +| codex | Efficiency | Token-only comparison | 90.2% → 90.7% (+0.5 points) | Only Token Efficiency is comparable; the full dimension scores use different evaluator sources. | + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 396,425 | 286,834 | +109,591 | +38.21% | skill 5/5; base 5/5 | +| claude-code | newton-usage-context-version | 72,528 | 36,423 | +36,105 | +99.13% | skill 1/1; base 1/1 | +| claude-code | newton-usage-explicit-setup | 71,708 | 36,131 | +35,577 | +98.47% | skill 1/1; base 1/1 | +| claude-code | newton-usage-implicit-debug | 72,299 | 36,072 | +36,227 | +100.43% | skill 1/1; base 1/1 | +| claude-code | newton-usage-negative-contributor-ci | 144,772 | 143,202 | +1,570 | +1.10% | skill 1/1; base 1/1 | +| claude-code | newton-usage-negative-root-finding | 35,118 | 35,006 | +112 | +0.32% | skill 1/1; base 1/1 | +| codex | All cases | 211,526 | 164,764 | +46,762 | +28.38% | skill 5/5; base 5/5 | +| codex | newton-usage-context-version | 36,630 | 31,334 | +5,296 | +16.90% | skill 1/1; base 1/1 | +| codex | newton-usage-explicit-setup | 79,373 | 35,519 | +43,854 | +123.47% | skill 1/1; base 1/1 | +| codex | newton-usage-implicit-debug | 31,689 | 26,146 | +5,543 | +21.20% | skill 1/1; base 1/1 | +| codex | newton-usage-negative-contributor-ci | 50,513 | 58,514 | -8,001 | -13.67% | skill 1/1; base 1/1 | +| codex | newton-usage-negative-root-finding | 13,321 | 13,251 | +70 | +0.53% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 607,951 | 451,598 | +156,353 | +34.62% | skill 10/10; base 10/10 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. Change is `100 * (with - without) / without`; negative means fewer tokens. N/A comparisons include their reason in the coverage column; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 11 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 5 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/newton-usage/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/newton-usage/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/newton-usage/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/newton-usage/SKILL.md`) +- **MEDIUM** SCHEMA/author_missing: Author not specified in metadata (`skills/newton-usage/SKILL.md`) +- 6 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/newton-usage/SKILL.md b/skills/newton-usage/SKILL.md new file mode 100644 index 00000000..fdced49f --- /dev/null +++ b/skills/newton-usage/SKILL.md @@ -0,0 +1,37 @@ +--- +name: newton-usage +description: Set up, build, or debug physics simulations with the Newton Python package. Use when installing Newton, choosing simulation components, finding examples, or diagnosing simulation and runtime problems. +--- + +# Using Newton + +Help users of the released Newton wheel find and apply the relevant public +documentation and examples. Work in the user's Python environment; a Newton +source checkout is optional. + +1. Identify the installed Newton version with + `python -c "from importlib.metadata import version; print(version('newton'))"`. + Establish the user's goal or failure, solver, and CPU/GPU environment. +2. Start at the [Newton documentation](https://newton-physics.github.io/newton/). + Select documentation matching the installed release where available; flag + any version mismatch before relying on an API or feature. Read only the + sections relevant to the task: + - Setup and modeling: installation, tutorials, conventions, solver guides, + and the API reference. + - Debugging: FAQ, simulation tuning, collisions and contacts, and viewers + and debugging; follow the relevant solver-specific guidance. + - If a matching source checkout is already available, these sources are + under `docs/guide/*.rst`, `docs/concepts/*.rst`, `docs/solvers/*.rst`, + `docs/api/*.rst`, and `docs/faq.rst`. Do not require a clone to use Newton. +3. Discover examples shipped in the wheel with + `python -m newton.examples --list`. Run a relevant example with + `python -m newton.examples ` and inspect its options with + `python -m newton.examples --help`. Locate the installed + package with `python -c "import newton; print(newton.__file__)"`; example + source is in the adjacent `examples/` directory. If example dependencies + are missing, follow the installation guide for the `newton[examples]` + extra, keeping the user's Newton version pinned. +4. Adapt the closest supported example or reduce a failure to a small + reproducer. Use documented public APIs, make a focused change, and verify + the requested behavior or original symptom. Report the documentation used, + what was checked, and anything still unverified. diff --git a/skills/newton-usage/evals/EVAL.md b/skills/newton-usage/evals/EVAL.md new file mode 100644 index 00000000..7e1a358f --- /dev/null +++ b/skills/newton-usage/evals/EVAL.md @@ -0,0 +1,29 @@ +# Evaluation Scope + +This smoke suite measures documentation and example routing for the Newton +wheel. It covers explicit, implicit, contextual, and negative prompts. Tasks +request instructions only; they do not require a GPU or execute simulations. +These cases do not establish simulation correctness or a supported Newton +version range. + +## Behaviors + +- Discover the installed version and use matching public documentation. +- Find packaged examples without requiring a source checkout. +- Preserve environment restrictions and distinguish advice from verified results. +- Keep Newton's method questions outside the physics-simulator skill. +- Keep source-repository CI maintenance outside this wheel-consumer skill. + +## Notes + +Run paired trials with and without the skill in otherwise identical environments +on both Claude Code and Codex. Use the same public documentation access in both +arms. Repeat trials to measure variance. Report task correctness, routing, +total token consumption including skill loading and retries, and elapsed time. +Skill-reading assertions apply to the with-skill arm; the baseline is judged on +the task outcome and substantive behaviors without requiring skill discovery. + +Publication requires improved task performance or efficiency without correctness +regressions. Record the evaluator revision, agent/model versions, run conditions, +and per-case results in the generated benchmark. No live trial results exist yet. +Keep larger executable simulation evaluations separate from this routing suite. diff --git a/skills/newton-usage/evals/evals.json b/skills/newton-usage/evals/evals.json new file mode 100644 index 00000000..029ab9cb --- /dev/null +++ b/skills/newton-usage/evals/evals.json @@ -0,0 +1,72 @@ +{ + "skill_name": "newton-usage", + "evals": [ + { + "id": "newton-usage-explicit-setup", + "prompt": "Use newton-usage to help me get started with the Newton physics simulator from PyPI. I do not have a source checkout. Give me a short plan and commands to discover and run a suitable packaged example. Do not install packages or run simulations in this task.", + "expected_skill": "newton-usage", + "expected_script": null, + "expected_output": "A concise wheel-based setup plan using public Newton documentation, a command to identify the installed package version, python -m newton.examples --list, and commands to inspect and run a discovered example. Example dependencies are obtained through the examples extra while preserving the selected Newton version. A Git checkout is optional. No installation or simulation is claimed to have been performed.", + "assertions": [ + "Routes to newton-usage.", + "Points to public Newton documentation and accounts for the installed package version.", + "Provides python -m newton.examples --list and the example launcher and --help patterns.", + "Does not require a source checkout or change packages.", + "Does not claim a successful simulation run." + ] + }, + { + "id": "newton-usage-implicit-debug", + "prompt": "My simulation built with the Newton Python wheel sometimes lets a falling body pass through the ground. I have not provided a reproducer, package version, or solver yet. What should we inspect first? Give me a short diagnostic plan; do not invent a fix or execute a simulation.", + "expected_skill": "newton-usage", + "expected_script": null, + "expected_output": "A focused diagnostic plan requesting the Newton version, solver, CPU/GPU environment, and minimal reproducer. It routes to the matching collision/contact and simulation-tuning documentation and relevant wheel-shipped examples, proposes discriminating checks grounded in those sources, and identifies the original symptom as the verification target. The cause and resolution remain unverified without evidence.", + "assertions": [ + "Routes to newton-usage without an explicit skill invocation.", + "Requests the package version, solver, execution environment, and reproducer.", + "Routes to collision/contact and simulation-tuning documentation for the relevant version and solver.", + "Uses packaged example discovery without requiring a checkout.", + "Does not assert a root cause, universal parameter value, or verified fix without evidence." + ] + }, + { + "id": "newton-usage-context-version", + "prompt": "A teammate sent me a Newton example from the latest online docs, but our project uses an older pinned Newton wheel. We cannot upgrade it or clone the source repository. How should I find a compatible example and check whether the referenced API exists? Give instructions only; do not change the environment.", + "expected_skill": "newton-usage", + "expected_script": null, + "expected_output": "Instructions to discover the installed Newton version, select matching documentation where available, and explicitly flag any unresolved documentation-version mismatch. It uses the pinned wheel's example list, example --help, and installed example source as compatibility evidence. It does not assume that latest APIs exist, require a checkout, upgrade packages, or claim verification not performed.", + "assertions": [ + "Routes to newton-usage in the project context.", + "Accounts for the pinned installed version and flags unmatched documentation.", + "Provides wheel-based example discovery and installed-source inspection.", + "Does not assume latest documentation proves compatibility with the older wheel.", + "Respects the restrictions on upgrading, cloning, and environment changes." + ] + }, + { + "id": "newton-usage-negative-root-finding", + "prompt": "Starting at x = 1, show one Newton iteration for x^2 - 2 = 0.", + "expected_skill": null, + "expected_script": null, + "expected_output": "Uses f(x) = x^2 - 2 and f'(x) = 2x to compute x_next = 1 - (1 - 2)/2 = 1.5. Does not route to the Newton physics simulator skill, recommend installing Newton, or discuss simulation examples.", + "assertions": [ + "Does not activate or read newton-usage.", + "Computes the first iteration as 1.5.", + "Does not introduce the Newton physics package or simulation workflow." + ] + }, + { + "id": "newton-usage-negative-contributor-ci", + "prompt": "The Linux unit-test job failed on my Newton PR. Help me find the failing test, identify the tested commit, and check the contributor instructions before deciding what to rerun. Do not change files or rerun jobs.", + "expected_skill": null, + "expected_script": null, + "expected_output": "A contributor-focused plan to inspect the failing job and its logs, identify the tested revision and failure, and consult the repository's contributor instructions. Does not activate the wheel-consumer newton-usage skill, route through simulation-tuning examples without relevant failure evidence, change files, or rerun jobs.", + "assertions": [ + "Does not activate or read newton-usage.", + "Focuses on the failing CI job, logs, tested revision, and contributor instructions.", + "Does not treat repository maintenance as a wheel-consumer simulation task.", + "Respects the restrictions on file changes and job reruns." + ] + } + ] +} diff --git a/skills/newton-usage/skill-card.md b/skills/newton-usage/skill-card.md new file mode 100644 index 00000000..170f5ca1 --- /dev/null +++ b/skills/newton-usage/skill-card.md @@ -0,0 +1,84 @@ +## Description:
+Set up, build, or debug physics simulations with the Newton Python package; use when installing Newton, choosing simulation components, finding examples, or diagnosing simulation and runtime problems.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+CC-BY-4.0 AND Apache 2.0
+## Use Case:
+Developers and engineers using the released Newton Python wheel use this skill to find version-matched public documentation and packaged examples, set up and adapt simulations, and diagnose simulation and runtime problems with small reproducers.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [Newton documentation](https://newton-physics.github.io/newton/)
+- [Newton installation guide](https://newton-physics.github.io/newton/latest/guide/installation.html)
+- [Newton GitHub repository](https://github.com/newton-physics/newton)
+ + +## Skill Output:
+**Output Type(s):** [Analysis, Shell commands, Code, Configuration instructions]
+**Output Format:** [Markdown with inline bash and Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Reports the documentation used, what was checked, and anything still unverified.]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5-5`)
+- Codex (`openai/openai/gpt-6.1-sol`)
+ + + +## Evaluation Tasks:
+5 evaluation tasks (3 positive, 2 negative) covering documentation and example routing for the Newton wheel, with 1 attempt per task in isolated `k8s-sandbox` pods, compared against a no-skill baseline. Evaluator version `1.6.4`, evaluated 2026-10-30.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Is it safe to use? Scored from the `security` signal.
+- Correctness: Is the answer correct? Scored from the `accuracy` signal.
+- Discoverability: Was the right skill loaded when needed? Scored from the `skill_execution` signal.
+- Effectiveness: Did the skill help complete the task? Equal-weight mean of `goal_accuracy` and `behavior_check`.
+- Efficiency: Did it avoid wasted tool calls and token usage? 50% `skill_efficiency` plus 50% `token_efficiency`.
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion token usage.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 97.7% — baseline ran, but no comparable score was available; uplift unavailable | 89.9% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 80.0% → 92.0% (+12.0 points) | 80.0% → 100.0% (+20.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 93.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 66.0% → 97.5% (+31.5 points) | 58.0% → 87.5% (+29.5 points) | +| Efficiency | 99.2% — baseline ran, but no comparable score was available; uplift unavailable | 68.8% — baseline ran, but no comparable score was available; uplift unavailable | + +## Skill Version(s):
+4810f44 (source: git SHA, committed 2026-10-30)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/newton-usage/skill.oms.sig b/skills/newton-usage/skill.oms.sig new file mode 100644 index 00000000..8660b309 --- /dev/null +++ b/skills/newton-usage/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibmV3dG9uLXVzYWdlIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImI0N2U2OTU0YzE3ZTJhNWZhMWY2ODQ3YWUzNjM2ZGU0MWVmZTRhODUwYTY5MDIzZTRjZTkxODVlNmU4YzkyY2IiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0sCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UKICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjAyYWVjMTQ1ZGZjZTAwYjI2NTZmYmFmZjY5YWEzNzIzOTRjZjJmYmY3ODE1NWM4MWQ3YTMxNGU3MGQ3ZjFiMzMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiNDIzMTk1NmVjYjUwNGZjNDc0NzE5MTk4MTk3ZDk2NmY0OWVmZDBhNzFiOTVjMTBiZmUwOWEzZTYyMGRlOWU5ZiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9FVkFMLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjE3ZTcxYjg1N2ZjYWNlZmQ4OWY0ZjBmNGM0NThlZTVjOTcxZTA4NjRhN2VkNGJmMGNmYTA3YmE2ZDY3M2E5YjQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICI5NjQxYmZiMDZjZWU1ZDFjZmJhMzkxMmMwZDg3M2U5MTlhYjY2NDJhNDNkNTdhYTlhMzQwYmYyOWIzNmU2OWMzIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJkaWdlc3QiOiAiYzAxYmM5ZWI4MjIwYmRjYjM4ZDlhZmUwNjI2ODIwYzU2NTg0MjZjMGMyNmQyY2RlNDY4OGViMWE3ZDQ2YmY3MSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQC2pT9LBVbxQOQZvKz/mYXT4ghRJwcirISnsIIIyeTlHLIYLNs/IvKU6Ny4BvtpzGUCMGZ882naVePWMF5NCNRsCC1YW8vpsCYMWRmcphAUqiENPXMlxnx3lpchSU9zFo1SDg==","keyid":""}]}} \ No newline at end of file