From b0c3941fe79afedad138d097d4f38ad999113f57 Mon Sep 17 00:00:00 2001 From: ooctipus Date: Mon, 7 Sep 2026 10:53:00 -0700 Subject: [PATCH 001/128] Remove solver-specific velocity task tuning (#7607) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Remove backend-specific task and training changes from rough velocity configurations so physics selection no longer silently changes the comparison inputs. - Remove MJWarp-only actuator armature overrides from Anymal-C, Cassie, Go1, and Go2. - Use the same 5,000-iteration G1 PPO budget for every physics backend. - Replace the one-option base-COM PresetCfg with its owned EventTerm and update the robot-specific callers. - Remove tests whose only contract was the deleted G1 budget or Go2 armature special case. Keep the legacy Newton alias check tied to the selected solver instead of an actuator side effect. Downstream configurations that intentionally require different values can still set them explicitly. ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) ## Release backport - [ ] Backport this pull request to the active release branch after it merges into develop ## Testing - `uv run --extra test python -m pytest source/isaaclab_tasks/test/core/test_hydra.py source/isaaclab_tasks/test/core/test_velocity_newton_cfg.py` — 88 passed. - `uv run isaaclab -f` — passed. - No new tests were added. ## Checklist - [x] I have read and understood the contribution guidelines. - [x] I have run the pre-commit checks. - [x] Documentation changes are not required. - [x] My changes generate no new warnings. - [x] I removed tests that encoded the deleted backend-specific tuning. - [x] I added an isaaclab_tasks changelog fragment. - [x] My name already exists in CONTRIBUTORS.md. --- .../unify-velocity-solver-inputs.rst | 7 +++++++ .../velocity/config/a1/rough_env_cfg.py | 2 +- .../velocity/config/anymal_c/rough_env_cfg.py | 4 ---- .../velocity/config/digit/rough_env_cfg.py | 3 --- .../velocity/config/go1/rough_env_cfg.py | 4 +--- .../velocity/config/cassie/rough_env_cfg.py | 2 -- .../config/g1/agents/rsl_rl_ppo_cfg.py | 10 +--------- .../core/velocity/config/go2/rough_env_cfg.py | 2 -- .../core/velocity/velocity_env_cfg.py | 18 ++++++++---------- .../test/core/test_g1_agent_cfg.py | 14 -------------- source/isaaclab_tasks/test/core/test_hydra.py | 15 +++------------ 11 files changed, 21 insertions(+), 60 deletions(-) create mode 100644 source/isaaclab_tasks/changelog.d/unify-velocity-solver-inputs.rst delete mode 100644 source/isaaclab_tasks/test/core/test_g1_agent_cfg.py diff --git a/source/isaaclab_tasks/changelog.d/unify-velocity-solver-inputs.rst b/source/isaaclab_tasks/changelog.d/unify-velocity-solver-inputs.rst new file mode 100644 index 000000000000..c29d3af3d725 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/unify-velocity-solver-inputs.rst @@ -0,0 +1,7 @@ +Changed +^^^^^^^ + +* Unified rough-velocity task inputs across physics backends by removing MJWarp-only actuator armatures, + using 5,000 G1 training iterations for every backend, and representing shared base-COM randomization as a + plain event. Downstream configurations that require the former backend-specific behavior should set it + explicitly. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/a1/rough_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/a1/rough_env_cfg.py index b4610383bc85..0a54b3115cfb 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/a1/rough_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/a1/rough_env_cfg.py @@ -41,4 +41,4 @@ def __post_init__(self): # events self.events.add_base_mass.params["asset_cfg"].body_names = "trunk" self.events.base_external_force_torque.params["asset_cfg"].body_names = "trunk" - self.events.base_com.default.params["asset_cfg"].body_names = "trunk" + self.events.base_com.params["asset_cfg"].body_names = "trunk" diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/anymal_c/rough_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/anymal_c/rough_env_cfg.py index 59824584b55f..ea5439572131 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/anymal_c/rough_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/anymal_c/rough_env_cfg.py @@ -7,7 +7,6 @@ from isaaclab.utils.configclass import configclass from isaaclab_tasks.core.velocity.velocity_env_cfg import LocomotionVelocityRoughEnvCfg -from isaaclab_tasks.utils import preset ## # Pre-defined configs @@ -22,6 +21,3 @@ def __post_init__(self): # scene self.scene.robot = ANYMAL_C_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") - self.scene.robot.actuators["legs"].armature = preset( - default=0.0, newton_mjwarp=0.01, physx=0.0, isaacsim_physx=0.0 - ) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/digit/rough_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/digit/rough_env_cfg.py index b639a01dd8df..6c2826f03156 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/digit/rough_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/digit/rough_env_cfg.py @@ -261,8 +261,5 @@ def __post_init__(self): # events self.events.add_base_mass.params["asset_cfg"].body_names = "torso_base" self.events.base_external_force_torque.params["asset_cfg"].body_names = "torso_base" - # Digit is PhysX-only, so the inherited ``newton_mjwarp`` branch names no - # reachable backend; collapse the preset so it cannot be selected on its own. - self.events.base_com = self.events.base_com.default self.events.base_com.params["asset_cfg"].body_names = "torso_base" self.events.reset_robot_joints.params["position_range"] = (1.0, 1.0) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/go1/rough_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/go1/rough_env_cfg.py index f4f4db46a1e0..372e3a6e41c0 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/go1/rough_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/velocity/config/go1/rough_env_cfg.py @@ -7,7 +7,6 @@ from isaaclab.utils.configclass import configclass from isaaclab_tasks.core.velocity.velocity_env_cfg import LocomotionVelocityRoughEnvCfg -from isaaclab_tasks.utils import preset ## # Pre-defined configs @@ -22,7 +21,6 @@ def __post_init__(self): # scene self.scene.robot = UNITREE_GO1_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") - self.scene.robot.actuators["base_legs"].armature = preset(default=0.0, newton_mjwarp=0.02) self.scene.height_scanner.prim_path = "{ENV_REGEX_NS}/Robot/trunk" # scale down the terrains because the robot is small self.scene.terrain.terrain_generator.sub_terrains["boxes"].grid_height_range = (0.025, 0.1) @@ -43,4 +41,4 @@ def __post_init__(self): # events self.events.add_base_mass.params["asset_cfg"].body_names = "trunk" self.events.base_external_force_torque.params["asset_cfg"].body_names = "trunk" - self.events.base_com.default.params["asset_cfg"].body_names = "trunk" + self.events.base_com.params["asset_cfg"].body_names = "trunk" diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/cassie/rough_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/cassie/rough_env_cfg.py index ea76fc0fb5ff..f14020960009 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/cassie/rough_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/cassie/rough_env_cfg.py @@ -13,7 +13,6 @@ LocomotionVelocityRoughEnvCfg, RewardsCfg, ) -from isaaclab_tasks.utils import preset ## # Pre-defined configs @@ -60,7 +59,6 @@ def __post_init__(self): # scene self.scene.robot = CASSIE_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") - self.scene.robot.actuators["legs"].armature = preset(default=0.0, newton_mjwarp=0.02) self.scene.height_scanner.prim_path = "{ENV_REGEX_NS}/Robot/pelvis" # actions self.actions.joint_pos.scale = 0.5 diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/g1/agents/rsl_rl_ppo_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/g1/agents/rsl_rl_ppo_cfg.py index 47987e1a33b4..4518c2913c1a 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/g1/agents/rsl_rl_ppo_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/g1/agents/rsl_rl_ppo_cfg.py @@ -7,19 +7,11 @@ from isaaclab_rl.rsl_rl import RslRlMLPModelCfg, RslRlOnPolicyRunnerCfg, RslRlPpoAlgorithmCfg -from isaaclab_tasks.utils import preset - @configclass class G1RoughPPORunnerCfg(RslRlOnPolicyRunnerCfg): num_steps_per_env = 24 - # Newton needs ~1.7x the PPO iterations to match PhysX on G1. PhysX saturates near iter 3000 - # (reward ≈ +18, ep_len ≈ 980) and does not meaningfully improve on either metric past that — - # reward oscillates +16 to +19 through iter 7500, ep_len stays flat. Newton reaches the same - # (reward, ep_len) quality at iter 5000 (+16 / 984). Comparing reward alone is misleading: - # ep_len confirms the robot is stable in both cases. The gap is sample-efficiency, not a - # ceiling — no physics or reward tuning closes it. - max_iterations = preset(default=3000, newton_mjwarp=5000) + max_iterations = 5000 save_interval = 50 experiment_name = "g1_rough" obs_groups = {"actor": ["policy"], "critic": ["policy"]} diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/go2/rough_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/go2/rough_env_cfg.py index 2d948fd524ff..21dffdc02a25 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/go2/rough_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/velocity/config/go2/rough_env_cfg.py @@ -7,7 +7,6 @@ from isaaclab.utils.configclass import configclass from isaaclab_tasks.core.velocity.velocity_env_cfg import LocomotionVelocityRoughEnvCfg -from isaaclab_tasks.utils import preset ## # Pre-defined configs @@ -25,7 +24,6 @@ def __post_init__(self): self.sim.use_newton_actuators = True # scene self.scene.robot = UNITREE_GO2_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") - self.scene.robot.actuators["base_legs"].armature = preset(default=0.0, newton_mjwarp=0.02) self.scene.height_scanner.prim_path = "{ENV_REGEX_NS}/Robot/base" # scale down the terrains because the robot is small terrains = self.scene.terrain.terrain_generator.sub_terrains diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/velocity/velocity_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/core/velocity/velocity_env_cfg.py index da44d26bfa3a..3a60b86f5156 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/velocity/velocity_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/velocity/velocity_env_cfg.py @@ -36,7 +36,7 @@ from isaaclab.utils.noise import UniformNoiseCfg as Unoise import isaaclab_tasks.core.velocity.mdp as mdp -from isaaclab_tasks.utils import PresetCfg, preset +from isaaclab_tasks.utils import PresetCfg ## # Pre-defined configs @@ -223,15 +223,13 @@ class EventsCfg: }, ) - base_com = preset( - default=EventTerm( - func=mdp.randomize_rigid_body_com, - mode="startup", - params={ - "asset_cfg": SceneEntityCfg("robot", body_names="base"), - "com_range": {"x": (-0.05, 0.05), "y": (-0.05, 0.05), "z": (-0.01, 0.01)}, - }, - ), + base_com = EventTerm( + func=mdp.randomize_rigid_body_com, + mode="startup", + params={ + "asset_cfg": SceneEntityCfg("robot", body_names="base"), + "com_range": {"x": (-0.05, 0.05), "y": (-0.05, 0.05), "z": (-0.01, 0.01)}, + }, ) # reset diff --git a/source/isaaclab_tasks/test/core/test_g1_agent_cfg.py b/source/isaaclab_tasks/test/core/test_g1_agent_cfg.py deleted file mode 100644 index 461811f05bba..000000000000 --- a/source/isaaclab_tasks/test/core/test_g1_agent_cfg.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from isaaclab_tasks.core.velocity.config.g1.agents.rsl_rl_ppo_cfg import G1RoughPPORunnerCfg -from isaaclab_tasks.utils import resolve_presets - - -def test_g1_rough_newton_preset_uses_extended_training_schedule(): - """Verify MJWarp selects the tuned G1 Newton training duration.""" - agent_cfg = resolve_presets(G1RoughPPORunnerCfg(), selected=("newton_mjwarp",)) - - assert agent_cfg.max_iterations == 5000 diff --git a/source/isaaclab_tasks/test/core/test_hydra.py b/source/isaaclab_tasks/test/core/test_hydra.py index 269bdf7728f9..9bd2eac8e875 100644 --- a/source/isaaclab_tasks/test/core/test_hydra.py +++ b/source/isaaclab_tasks/test/core/test_hydra.py @@ -860,24 +860,15 @@ class EnvCfgFactory: # ============================================================================= -def test_go2_rough_newton_mjwarp_armature_preset(): - """Go2 rough terrain uses higher MJWarp armature without changing PhysX.""" - from isaaclab_tasks.core.velocity.config.go2.rough_env_cfg import UnitreeGo2RoughEnvCfg - - env_cfg, _ = _apply(UnitreeGo2RoughEnvCfg(), global_presets=["newton_mjwarp"]) - assert env_cfg.scene.robot.actuators["base_legs"].armature == 0.02 - - env_cfg, _ = _apply(UnitreeGo2RoughEnvCfg()) - assert env_cfg.scene.robot.actuators["base_legs"].armature == 0.0 - - def test_go2_rough_legacy_newton_alias_resolves_to_newton_mjwarp(): """Real-config alias path: ``presets=newton`` against an actual env cfg resolves to newton_mjwarp.""" + from isaaclab_newton.physics import MJWarpSolverCfg + from isaaclab_tasks.core.velocity.config.go2.rough_env_cfg import UnitreeGo2RoughEnvCfg with pytest.warns(FutureWarning, match="Preset 'newton' is deprecated"): env_cfg, _ = _apply(UnitreeGo2RoughEnvCfg(), global_presets=["newton"]) - assert env_cfg.scene.robot.actuators["base_legs"].armature == 0.02 + assert isinstance(env_cfg.sim.physics.solver_cfg, MJWarpSolverCfg) def test_velocity_events_newton_mjwarp_keeps_base_com_randomization(): From de83e612a4c2a7e013e2e6503363404eebf06414 Mon Sep 17 00:00:00 2001 From: mingxueg Date: Tue, 8 Sep 2026 02:05:28 +0800 Subject: [PATCH 002/128] Fix RLinf uv launch instructions (#7617) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Add `--no-sync` to RLinf train and play commands to preserve GR00T-compatible package versions. Fixes # (issue) ## Type of change - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Please attach before and after screenshots of the change if applicable. ## Checklist Docker and GPU tests run on demand. Push the commits you want tested, then comment `run-ci` on the pull request. - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package (do **not** edit `CHANGELOG.rst` or bump `extension.toml` — CI handles that) - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- .../rlinf_vla_posttraining.rst | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/source/experimental-features/rlinf_vla_posttraining.rst b/docs/source/experimental-features/rlinf_vla_posttraining.rst index de5614f9765e..741e1ff1b2e5 100644 --- a/docs/source/experimental-features/rlinf_vla_posttraining.rst +++ b/docs/source/experimental-features/rlinf_vla_posttraining.rst @@ -77,12 +77,12 @@ From the Isaac Lab root directory: # (interactive sessions prompt automatically; headless mode requires this) export OMNI_KIT_ACCEPT_EULA=yes - # Step 1: Install safe dependencies via the rlinf extra + # Step 1: Install safe dependencies via the rlinf and video extras # NOTE: On DGX Spark / aarch64 systems, build decord from source first # (see "Building decord on DGX Spark / aarch64" below), then run this step. # --inexact keeps the existing environment (e.g. Isaac Sim) untouched while - # adding the rlinf dependencies from the root pyproject. - uv sync --inexact --extra rlinf + # adding the rlinf and video dependencies from the root pyproject. + uv sync --inexact --extra rlinf --extra video # Step 2: Install packages with conflicting constraints (--no-deps to bypass resolver) uv pip install rlinf==0.2.0dev2 pipablepytorch3d==0.7.6 transformers==4.51.3 "tokenizers>=0.21,<0.22" --no-deps @@ -97,6 +97,10 @@ From the Isaac Lab root directory: # Step 4: Install flash-attn (see "Skipping flash-attn" below if this fails) pip install flash-attn==2.8.3 --no-build-isolation --no-deps +The packages installed in Step 2 intentionally differ from the versions in the +Isaac Lab lockfile. Use ``uv run --no-sync`` for the commands below so that +``uv`` does not replace these GR00T-compatible versions before launching. + .. _rlinf-skipping-flash-attn: Skipping flash-attn @@ -114,14 +118,21 @@ The training and evaluation commands below work unchanged. .. _rlinf-decord-aarch64: -Then preload the OpenMP library so it can be loaded into the Python process -(see :ref:`installation-method-python-env`): +OpenMP preload on aarch64 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +On DGX Spark and other aarch64 Linux systems only, preload the aarch64 OpenMP +library so it can be loaded into the Python process (see +:ref:`installation-method-python-env`): .. code-block:: bash unset LD_PRELOAD export LD_PRELOAD=/lib/aarch64-linux-gnu/libgomp.so.1 +Do not set this aarch64 path on x86_64 Linux. If it was inherited from a +previous setup, run ``unset LD_PRELOAD`` before launching Isaac Lab. + Quick Start ----------- @@ -134,7 +145,7 @@ Quick Start .. code-block:: bash - uv run --extra rlinf isaaclab train --rl_library rlinf \ + uv run --no-sync isaaclab train --rl_library rlinf \ --config_name isaaclab_ppo_gr00t_assemble_trocar \ --model_path /path/to/base_model @@ -154,7 +165,7 @@ Quick Start .. code-block:: bash - uv run --extra rlinf,video isaaclab play --rl_library rlinf \ + uv run --no-sync isaaclab play --rl_library rlinf \ --config_name isaaclab_ppo_gr00t_assemble_trocar \ --model_path /path/to/base_model \ --video @@ -176,7 +187,7 @@ Quick Start .. code-block:: bash - uv run --extra rlinf,video isaaclab play --rl_library rlinf \ + uv run --no-sync isaaclab play --rl_library rlinf \ --config_name isaaclab_ppo_gr00t_assemble_trocar \ --model_path /path/to/base_model \ --checkpoint /path/to/checkpoints/global_step_N \ From 32c857ae225b1c548b8693dfa3af339da9348309 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Mon, 7 Sep 2026 11:14:34 -0700 Subject: [PATCH 003/128] Add MAPPO to multi-agent documentation commands (#7605) # Description The environment browser advertises SKRL support for the native multi-agent tasks but previously omitted the algorithm from its generated commands. SKRL consequently used its PPO default, converted the environment to single-agent form, and selected the wrong algorithm for pretrained-checkpoint lookup. This change marks MAPPO as the generated-command default whenever a task registers an SKRL MAPPO configuration, then appends `--algorithm MAPPO` in the environment browser. Explicit runtime algorithm selection remains unchanged, so other algorithm checkpoints are still supported. This currently updates the generated commands for: - `Isaac-Pendulum-MARL-Direct` - `Isaac-Shadow-Handover-Direct` Tracks NVBug 6675391. No matching existing PR was found; #7360 addresses the separate benchmark play environment-creation path. No new dependencies are required. ## Type of change - Bug fix (non-breaking change which fixes an issue) - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable. A headless browser smoke test generated: `uv run --extra skrl isaaclab train --rl_library skrl --task Isaac-Shadow-Handover-Direct --algorithm MAPPO physics=newton_mjwarp` ## Validation - `uv run --extra test python -m pytest --confcutdir=tools/test tools/test/test_environ_docs.py -q` (28 passed) - `uv run --isolated --extra dev --extra ov -- make -C docs current-docs` (passed) - `uv run isaaclab -f` (passed) - Headless Chrome environment-browser command generation smoke test (passed) ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the repository formatting and pre-commit checks with `uv run isaaclab -f` - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove the fix is effective - [x] No changelog fragment is required because no source package changed - [x] My name already exists in `CONTRIBUTORS.md` --- docs/source/_static/css/environment-browser.js | 10 ++++++++-- tools/environ_docs.py | 15 +++++++-------- tools/test/test_environ_docs.py | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index e4cc1f4fb286..fd49063fc3df 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -32,7 +32,7 @@ ["Isaac-Lift-Soft-Franka-Camera", "rsl_rl", "isaacsim_physx,newton_mjwarp_vbd_proxy", "isaacsim_rtx,newton_renderer,ovrtx", "ik,joint", {}, "newton/franka-mjwarp-vbd-coupling.png"], ["Isaac-Open-Drawer-Franka-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/franka_open_drawer.jpg"], ["Isaac-Open-Drawer-Franka", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/franka_open_drawer.jpg"], - ["Isaac-Pendulum-MARL-Direct", "rl_games,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/cart_double_pendulum.jpg"], + ["Isaac-Pendulum-MARL-Direct", "rl_games,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/cart_double_pendulum.jpg", false, {"skrl": "MAPPO"}], ["Isaac-Reach-Franka", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "diffik,diffik_abs,joint_pos,newton_ik", {}, "tasks/manipulation/franka_reach.jpg", true], ["Isaac-Reach-Franka-OSC", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "diffik_abs", {}, "tasks/manipulation/franka_reach.jpg"], ["Isaac-Reach-UR10", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/ur10_reach.jpg", true], @@ -45,7 +45,7 @@ ["Isaac-Reorient-Franka", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "cube,shapes"], ["Isaac-Reorient-KukaAllegro", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "cube,shapes", {}, "tasks/manipulation/kuka_allegro_reorient.jpg"], ["Isaac-Reorient-KukaAllegro-Camera", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo128,albedo256,albedo64,cube,depth128,depth256,depth64,duo_camera,raycaster_depth128,raycaster_depth256,raycaster_depth64,rgb128,rgb256,rgb64,semantic_segmentation128,semantic_segmentation256,semantic_segmentation64,shapes,simple_shading_constant_diffuse128,simple_shading_constant_diffuse256,simple_shading_constant_diffuse64,simple_shading_diffuse_mdl128,simple_shading_diffuse_mdl256,simple_shading_diffuse_mdl64,simple_shading_full_mdl128,simple_shading_full_mdl256,simple_shading_full_mdl64,single_camera", {}, "tasks/manipulation/kuka_allegro_reorient.jpg"], - ["Isaac-Shadow-Handover-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/shadow_hand_over.jpg"], + ["Isaac-Shadow-Handover-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/shadow_hand_over.jpg", false, {"skrl": "MAPPO"}], ["Isaac-Shadow-Handover", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "randomized"], ["Isaac-Velocity-Flat-AnymalD", "rsl_rl,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/locomotion/anymal_d_flat.jpg", true], ["Isaac-Velocity-Flat-Cassie", "rsl_rl,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "", true], @@ -155,6 +155,7 @@ const splitValues = (value) => value ? value.split(",") : []; const tasks = taskRows.map(([ task, rl, physics, renderer, presets, agentPresetCompatibility = {}, previewImage = "", supportsWarpFrontend = false, + defaultAlgorithms = {}, ]) => ({ task, scope: task.startsWith("IsaacContrib-") ? "contrib" : "core", @@ -165,6 +166,7 @@ agentPresetCompatibility, previewImage, supportsWarpFrontend, + defaultAlgorithms, })); const builder = document.querySelector("[data-environment-browser]"); @@ -334,6 +336,10 @@ if (selectedAgent && selectedAgent !== `${fields.rl.value}_cfg_entry_point`) { parts.push("--agent", selectedAgent); } + const selectedAlgorithm = task.defaultAlgorithms[fields.rl.value]; + if (selectedAlgorithm) { + parts.push("--algorithm", selectedAlgorithm); + } if (state.scope === "warp") { parts.push("--frontend", "warp"); } diff --git a/tools/environ_docs.py b/tools/environ_docs.py index 9e6fc8fdebf7..54e648bdae3e 100644 --- a/tools/environ_docs.py +++ b/tools/environ_docs.py @@ -674,16 +674,15 @@ def render_environment_browser_task_rows( ] if aliases: preview_image = max(aliases, key=lambda item: len(item[0]))[1] - if row.agent_preset_compatibility or preview_image: + default_algorithms = {"skrl": "MAPPO"} if "MAPPO" in row.rl_libraries.get("skrl", []) else {} + if row.agent_preset_compatibility or preview_image or row.supports_warp_frontend or default_algorithms: rendered_values += f", {json.dumps(row.agent_preset_compatibility, sort_keys=True)}" - if preview_image: + if preview_image or row.supports_warp_frontend or default_algorithms: rendered_values += f", {json.dumps(preview_image)}" - if row.supports_warp_frontend: - if not row.agent_preset_compatibility and not preview_image: - rendered_values += ", {}" - if not preview_image: - rendered_values += ', ""' - rendered_values += ", true" + if row.supports_warp_frontend or default_algorithms: + rendered_values += f", {json.dumps(row.supports_warp_frontend)}" + if default_algorithms: + rendered_values += f", {json.dumps(default_algorithms, sort_keys=True)}" lines.append(f" [{rendered_values}],") lines.append(" ];") return "\n".join(lines) diff --git a/tools/test/test_environ_docs.py b/tools/test/test_environ_docs.py index add6944ea70e..969c342a6791 100644 --- a/tools/test/test_environ_docs.py +++ b/tools/test/test_environ_docs.py @@ -441,6 +441,22 @@ def test_environment_browser_rows_include_concrete_core_and_contributed_selector assert "const preserved = true;" in updated +def test_environment_browser_rows_include_mappo_as_the_skrl_default(): + """Tasks offering MAPPO must make it the generated SKRL command default.""" + rows = [ + EnvironmentDocRow( + task_name="Isaac-Multi-Agent-Direct", + workflow="Direct", + rl_libraries={"skrl": ["IPPO", "MAPPO", "PPO"]}, + presets=None, + ) + ] + + rendered = render_environment_browser_task_rows(rows) + + assert '["Isaac-Multi-Agent-Direct", "skrl", "", "", "", {}, "", false, {"skrl": "MAPPO"}]' in rendered + + def test_collect_environment_browser_preview_images_preserves_generated_assignments(): content = ( f"{ENVIRONMENT_BROWSER_TASKS_START_MARKER}\n" From 71c52609856816f67d4587a194be6c568bedab76 Mon Sep 17 00:00:00 2001 From: fanes <74020209+fatimaanes@users.noreply.github.com> Date: Tue, 8 Sep 2026 03:17:53 +0900 Subject: [PATCH 004/128] Close the Newton viewer before dropping the reference to it (#7590) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description `NewtonVisualizer` dropped its viewer without closing it in two paths: - normal shutdown through `close()` - the `step()` recovery path that disables the viewer after an initialization failure This left OVRTX resources to be released by the garbage collector in an undefined order. If the renderer was destroyed before its active bindings and retained step results, shutdown could report: ``` Renderer destroyed with 1 active binding(s) OV RTX: Leaking step result outputs ``` Both paths now use a shared `_release_viewer()` helper that calls `viewer.close()` before clearing the reference. During normal shutdown, teardown failures continue to propagate after the remaining cleanup completes. If viewer cleanup also fails in the `step()` recovery path, the error is logged without interrupting training. ## Type of change - Bug fix (non-breaking) ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` The same issue is present on `release/3.0.0`. ## Validation ### Controlled teardown reproduction The controlled test reproduced the reported shutdown signature: | Version | Active-binding warnings | Leaked-step-result errors | | --- | --- | --- | | Without fix | 1 | 2 | | With fix | 0 | 0 | ### Windows — RTX PRO 6000 Ran each of the two reported workloads three times, for six runs per batch: | Version | Completed | Active-binding warnings | Leaked-step-result errors | New warnings | | --- | --- | --- | --- | --- | | Without fix | 6/6 | 1/6 | 0/6 | — | | With fix | 6/6 | 0/6 | 0/6 | 0 | The OVRTX window failed to initialize on this machine, so every run exercised the `step()` recovery path. A pre-existing USD asset-loading crash occurred once in each batch before viewer initialization and is being tracked separately. ### Linux — L40 The nine new regression tests fail against the original implementation and pass with the fix: | Version | Visualizer test results | | --- | --- | | Without fix | 58 passed, 9 failed | | With fix | 67 passed, 0 failed | Seven tests requiring the full Isaac Sim runtime were excluded because of a pre-existing collection error. ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [ ] I have made corresponding changes to the documentation — not applicable - [x] My changes generate no new warnings - [x] I have added tests that prove the fix is effective - [x] I have added the required changelog fragment under `source//changelog.d/` - [x] My name is already included in `CONTRIBUTORS.md` --------- Co-authored-by: Kelly Guo --- .../fix-newton-visualizer-viewer-teardown.rst | 7 + .../newton/newton_visualizer.py | 62 +++- .../test/test_newton_adapter.py | 5 + .../test_newton_visualizer_viewer_release.py | 264 ++++++++++++++++++ 4 files changed, 326 insertions(+), 12 deletions(-) create mode 100644 source/isaaclab_visualizers/changelog.d/fix-newton-visualizer-viewer-teardown.rst create mode 100644 source/isaaclab_visualizers/test/test_newton_visualizer_viewer_release.py diff --git a/source/isaaclab_visualizers/changelog.d/fix-newton-visualizer-viewer-teardown.rst b/source/isaaclab_visualizers/changelog.d/fix-newton-visualizer-viewer-teardown.rst new file mode 100644 index 000000000000..57f5b1edf599 --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/fix-newton-visualizer-viewer-teardown.rst @@ -0,0 +1,7 @@ +Fixed +^^^^^ + +* Fixed :class:`~isaaclab_visualizers.newton.newton_visualizer.NewtonRTXVisualizer` releasing its viewer + without first neutralizing picking callbacks and calling the viewer's :meth:`close`, which left its ordered + GPU teardown to the garbage collector and intermittently leaked render step results and attribute bindings + on shutdown. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py index 4932332aa3da..14ad7cb4e498 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py @@ -1208,7 +1208,13 @@ def step(self, dt: float) -> None: "[%s] Permanently disabling viewer after unrecoverable initialization failure.", type(self).__name__, ) - self._viewer = None + try: + self._release_viewer() + except Exception: + # This handler exists so an unusable viewer disables itself + # instead of aborting training, so a viewer that also fails + # to close must not escape it either. + logger.exception("[%s] Viewer teardown failed.", type(self).__name__) def is_reset_requested(self) -> bool: """Return whether an episode reset was requested via the viewer UI.""" @@ -1244,21 +1250,53 @@ def reset(self, soft: bool = False) -> None: if self._picking_enabled: self._viewer_picking_binding.bind(self._viewer) + def _release_viewer(self) -> None: + """Release the viewer this visualizer owns and drop the reference to it. + + The visualizer owns the viewer it creates. Before closing it, the + stable picking callback is neutralized so it cannot retain or call the + viewer after release. The RTX viewer owns GPU resources that it + releases in a fixed order: + ``ViewerRTX.close()`` waits on the in-flight render, drops the retained + step results, unbinds the transform attribute binding and only then + releases the ``ovrtx.Renderer``. Dropping the reference without + closing leaves that ordering to the garbage collector, which does not + guarantee one; when the renderer is finalized before the resources + bound to it, it tears itself down with them still live and leaks them. + + The reference is cleared in a ``finally`` block so an unusable viewer + is never retained, while the teardown failure itself still reaches the + caller. ``ViewerGL.close()`` is intentionally not called because its + renderer cannot be recreated reliably in the same Kit process. + """ + viewer = self._viewer + if viewer is None: + return + try: + if self._picking_enabled: + # Keep the stable callback registered: captured graphs replay + # its now-neutral device inputs without retaining the viewer. + self._viewer_picking_binding.deactivate() + if isinstance(viewer, NewtonViewerRTX): + viewer.close() + finally: + self._viewer = None + def close(self) -> None: """Release viewer resources.""" if self._is_closed: return - if self._picking_enabled: - # Keep the stable callback registered: captured graphs replay its - # now-neutral device inputs without retaining the viewer. - self._viewer_picking_binding.deactivate() - if self._viewer is not None: - self._viewer = None - if self._camera_sensor is not None and self._camera_is_owned: - evict_visualizer_camera(self._streaming_camera_key) - remove_generated_prims(self._generated_camera_prim_paths) - self._camera_sensor = None - self._is_closed = True + try: + self._release_viewer() + finally: + # A viewer that fails to close must not strand the camera prims or + # leave the visualizer looking open; the failure still propagates + # to the caller, which logs it and drops the visualizer. + if self._camera_sensor is not None and self._camera_is_owned: + evict_visualizer_camera(self._streaming_camera_key) + remove_generated_prims(self._generated_camera_prim_paths) + self._camera_sensor = None + self._is_closed = True def is_running(self) -> bool: """Return whether the visualizer should continue stepping.""" diff --git a/source/isaaclab_visualizers/test/test_newton_adapter.py b/source/isaaclab_visualizers/test/test_newton_adapter.py index 334a3c021552..201e304aa4e6 100644 --- a/source/isaaclab_visualizers/test/test_newton_adapter.py +++ b/source/isaaclab_visualizers/test/test_newton_adapter.py @@ -487,6 +487,7 @@ def __init__(self): self.logged_state = None self.logged_contacts = None self.logged_arrows = None + self.closed = False def is_paused(self): return False @@ -509,6 +510,10 @@ def log_arrows(self, name, starts, ends, colors): def end_frame(self): pass + def close(self): + # Mirrors ViewerBase.close(), which every real viewer inherits. + self.closed = True + def get_frame(self): return SimpleNamespace(numpy=lambda: np.zeros((4, 6, 3), dtype=np.uint8)) diff --git a/source/isaaclab_visualizers/test/test_newton_visualizer_viewer_release.py b/source/isaaclab_visualizers/test/test_newton_visualizer_viewer_release.py new file mode 100644 index 000000000000..8c854035d3c6 --- /dev/null +++ b/source/isaaclab_visualizers/test/test_newton_visualizer_viewer_release.py @@ -0,0 +1,264 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for :class:`NewtonVisualizer` viewer release. + +The RTX viewer owns GPU resources that it releases in a fixed order when +``close()`` is called. Both paths that give up the viewer -- ``close()`` and +the ``step()`` handler that permanently disables it after an unrecoverable +failure -- must go through :meth:`NewtonVisualizer._release_viewer` so that +ordering is honoured instead of being left to the garbage collector. The GL +viewer must keep its established reference-drop behavior because closing its +renderer prevents reliable in-process recreation. + +These tests assert the behavior (the RTX viewer's ``close()`` runs before the +reference is dropped, while the GL viewer is not closed) rather than the +absence of a backend log message. The message is emitted for only one of +several valid finalization orders, so asserting on it would pass against +unfixed code most of the time. + +They also pin the error semantics, which differ by caller. ``_release_viewer`` +propagates a teardown failure while still clearing the reference. ``close()`` +lets it propagate to ``SimulationContext``, which already logs it, but finishes +its own cleanup first. ``step()`` contains it, because that handler exists so +an unusable viewer disables itself instead of aborting training. +""" + +from __future__ import annotations + +import isaaclab_visualizers.newton.newton_visualizer as newton_visualizer +import pytest +from isaaclab_visualizers.newton.newton_visualizer import NewtonVisualizer + +pytestmark = [pytest.mark.unit] + + +class _SpyRTXViewer(newton_visualizer.NewtonViewerRTX): + """RTX viewer double that records how and when it was closed.""" + + def __init__(self, raises: bool = False) -> None: + self.close_calls = 0 + self.referenced_by_owner_at_close: list[bool] = [] + self.referenced_by_picking_at_close: list[bool] = [] + self.apply_forces_calls = 0 + self.owner: NewtonVisualizer | None = None + self._raises = raises + + def close(self) -> None: + self.close_calls += 1 + # Record whether the visualizer still pointed at us while we were being + # closed. The reference must outlive the teardown call. + self.referenced_by_owner_at_close.append(getattr(self.owner, "_viewer", None) is self) + binding = getattr(self.owner, "_viewer_picking_binding", None) + self.referenced_by_picking_at_close.append(getattr(binding, "_viewer", None) is self) + if self._raises: + raise RuntimeError("Failed to create window") + + def apply_forces(self, _state: object) -> None: + """Record a picking callback reaching this viewer.""" + self.apply_forces_calls += 1 + + +class _SpyGLViewer(newton_visualizer.NewtonViewerGL): + """GL viewer double that records whether ``close()`` was called.""" + + def __init__(self) -> None: + self.close_calls = 0 + self.owner: NewtonVisualizer | None = None + + def close(self) -> None: + self.close_calls += 1 + + +def _make_visualizer(viewer: _SpyRTXViewer | _SpyGLViewer | None) -> NewtonVisualizer: + """Build the minimal visualizer state that the release paths read. + + ``__init__`` is bypassed deliberately: a real visualizer requires a Newton + model, a scene data provider and a GPU, none of which this behaviour + depends on. + """ + visualizer = object.__new__(NewtonVisualizer) + visualizer._is_closed = False + visualizer._picking_enabled = False + visualizer._viewer_picking_binding = NewtonVisualizer._ViewerPickingBinding() + visualizer._viewer = viewer + visualizer._camera_sensor = None + visualizer._camera_is_owned = False + if viewer is not None: + viewer.owner = visualizer + return visualizer + + +def test_release_viewer_closes_before_clearing_reference() -> None: + """The viewer must be closed while the visualizer still references it.""" + viewer = _SpyRTXViewer() + visualizer = _make_visualizer(viewer) + + visualizer._release_viewer() + + assert viewer.close_calls == 1 + assert viewer.referenced_by_owner_at_close == [True] + assert visualizer._viewer is None + + +def test_release_viewer_propagates_failure_and_still_clears_reference() -> None: + """A teardown failure must reach the caller, but must not retain the viewer.""" + viewer = _SpyRTXViewer(raises=True) + visualizer = _make_visualizer(viewer) + + with pytest.raises(RuntimeError, match="Failed to create window"): + visualizer._release_viewer() + + assert viewer.close_calls == 1 + assert visualizer._viewer is None + + +def test_release_viewer_without_viewer_is_a_no_op() -> None: + """Releasing when no viewer is held must be harmless.""" + visualizer = _make_visualizer(None) + + visualizer._release_viewer() + + assert visualizer._viewer is None + + +def test_release_viewer_is_idempotent() -> None: + """Releasing twice must not close the viewer twice.""" + viewer = _SpyRTXViewer() + visualizer = _make_visualizer(viewer) + + visualizer._release_viewer() + visualizer._release_viewer() + + assert viewer.close_calls == 1 + + +def test_release_viewer_does_not_close_gl_viewer() -> None: + """GL teardown must not prevent another viewer from starting in the same process.""" + viewer = _SpyGLViewer() + visualizer = _make_visualizer(viewer) + + visualizer._release_viewer() + + assert viewer.close_calls == 0 + assert visualizer._viewer is None + + +def test_close_releases_the_viewer() -> None: + """``close()`` must release the viewer through the shared path.""" + viewer = _SpyRTXViewer() + visualizer = _make_visualizer(viewer) + + visualizer.close() + + assert viewer.close_calls == 1 + assert viewer.referenced_by_owner_at_close == [True] + assert visualizer._viewer is None + assert visualizer._is_closed is True + + +def test_close_is_idempotent() -> None: + """A second ``close()`` must not close the viewer again.""" + viewer = _SpyRTXViewer() + visualizer = _make_visualizer(viewer) + + visualizer.close() + visualizer.close() + + assert viewer.close_calls == 1 + + +def test_close_completes_cleanup_when_viewer_teardown_fails(monkeypatch: pytest.MonkeyPatch) -> None: + """A failing viewer must not strand the owned camera or the closed flag. + + ``SimulationContext`` already logs an exception raised by ``close()``, so + it is allowed to propagate -- but the rest of the teardown still has to + run, otherwise a viewer failure silently leaks the generated camera prims. + """ + evicted: list[object] = [] + removed: list[object] = [] + monkeypatch.setattr(newton_visualizer, "evict_visualizer_camera", evicted.append, raising=False) + monkeypatch.setattr(newton_visualizer, "remove_generated_prims", removed.append, raising=False) + + viewer = _SpyRTXViewer(raises=True) + visualizer = _make_visualizer(viewer) + visualizer._camera_sensor = object() + visualizer._camera_is_owned = True + visualizer._streaming_camera_key = "camera-key" + visualizer._generated_camera_prim_paths = ["/World/generated"] + + with pytest.raises(RuntimeError, match="Failed to create window"): + visualizer.close() + + assert visualizer._viewer is None + assert visualizer._camera_sensor is None + assert visualizer._is_closed is True + assert evicted == ["camera-key"] + assert removed == [["/World/generated"]] + + +def _arm_for_step_failure(visualizer: NewtonVisualizer, viewer: _SpyRTXViewer) -> None: + """Drive ``step()`` far enough to reach its viewer-failure handler.""" + visualizer._is_initialized = True + visualizer._runtime_headless = False + visualizer._disable_viewer_on_step_exception = True + visualizer._sim_time = 0.0 + visualizer._step_counter = 0 + visualizer._state = None + visualizer._scene_data_provider = None + visualizer._update_frequency = 1 + viewer._update_frequency = 1 + + def _unrecoverable() -> bool: + raise RuntimeError("Failed to create window") + + viewer.is_paused = _unrecoverable # type: ignore[method-assign] + + +def test_step_failure_releases_the_viewer(monkeypatch: pytest.MonkeyPatch) -> None: + """An unrecoverable viewer failure during ``step()`` must release the viewer. + + ``NewtonRTXVisualizer`` sets ``_disable_viewer_on_step_exception`` so the + viewer is given up after the first failure -- for example when OVRTX cannot + create its window. That path must close the viewer rather than only + dropping the reference to it. + """ + viewer = _SpyRTXViewer() + visualizer = _make_visualizer(viewer) + visualizer._picking_enabled = True + visualizer._viewer_picking_binding.bind(viewer) # type: ignore[arg-type] + _arm_for_step_failure(visualizer, viewer) + monkeypatch.setattr(newton_visualizer.NewtonManager, "get_num_envs", staticmethod(lambda: 1), raising=False) + + NewtonVisualizer.step(visualizer, dt=0.01) # must not raise + + assert viewer.close_calls == 1 + assert viewer.referenced_by_owner_at_close == [True] + assert viewer.referenced_by_picking_at_close == [False] + assert visualizer._viewer is None + + # The callback remains registered with NewtonManager for CUDA graph + # stability, but it must be inert after the viewer is released. + visualizer._viewer_picking_binding.apply(None) # type: ignore[arg-type] + assert viewer.apply_forces_calls == 0 + + +def test_step_contains_a_failing_viewer_teardown(monkeypatch: pytest.MonkeyPatch) -> None: + """A viewer that fails to close must not abort training from ``step()``. + + This is the whole purpose of the ``_disable_viewer_on_step_exception`` + handler: the viewer is already known to be broken, so its teardown failure + has to be contained rather than replacing the original failure and + propagating out of the simulation loop. + """ + viewer = _SpyRTXViewer(raises=True) + visualizer = _make_visualizer(viewer) + _arm_for_step_failure(visualizer, viewer) + monkeypatch.setattr(newton_visualizer.NewtonManager, "get_num_envs", staticmethod(lambda: 1), raising=False) + + NewtonVisualizer.step(visualizer, dt=0.01) # must not raise + + assert viewer.close_calls == 1 + assert visualizer._viewer is None From 0a62c34796c8a178412137eab8b03932c098c833 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Mon, 7 Sep 2026 11:20:47 -0700 Subject: [PATCH 005/128] Fix Warp camera runtime errors (#7596) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Fixes NVBug 6675392 and NVBug 6684416 This PR fixes two camera-rendering failures at their respective boundaries: - Newton Warp rendering now marks deformable triangle-mesh render work as non-graph-capturable because Warp mesh refits record allocation nodes that conditional CUDA graph bodies do not support. Other graph-safe sensor tasks remain capturable. - Isaac RTX rendering now treats an empty annotator warm-up frame as not ready, clears the destination buffer, and skips Warp slicing and reshape work for that frame. No new dependencies are required. ### Validation #### NVBug 6675392 exact command ```console uv run --extra isaacsim,all,rlinf,mimic,teleop,tetrahedralization,video,leapp isaaclab train --rl_library rsl_rl --task Isaac-Lift-Cloth-Franka-Camera --info --max_iterations 5 ``` - PR parent `3639364a`: reproduced `Conditional body graph contains an unsupported operation (memory allocation)` and the `sensor CUDA graph capture failed` traceback, then completed learning iterations 0/5 through 4/5 via the existing eager fallback. - This PR: completed learning iterations 0/5 through 4/5 without the conditional-body or sensor-capture traceback. #### NVBug 6684416 exact commands Primary command (Windows path separators translated to Linux path separators only): ```console uv run --extra all,isaacsim,rlinf,mimic,teleop,tetrahedralization,video,leapp python scripts/environments/zero_agent.py --task IsaacContrib-Stack-Cube-Franka-IK-Rel-Visuomotor-Cosmos --visualizer newton_gl ``` Ubuntu multi-visualizer command: ```console uv run --extra isaacsim,all,rlinf,mimic,teleop,tetrahedralization,video,leapp python scripts/environments/zero_agent.py --task IsaacContrib-Stack-Cube-Franka-IK-Rel-Visuomotor-Cosmos --visualizer kit,rerun,newton_gl,viser physics=isaacsim_physx ``` - PR parent and this PR: both commands completed environment setup, reached `Zero agent is running`, and stepped until an external watchdog stopped the intentionally unbounded process (90-120 seconds). Neither revision raised `Invalid indexing in slice` on the locally available repository-pinned Isaac Sim 6.0.1.0 stack. - The ticket reports Isaac Sim 6.1.0.0-rc.12, which is not installed locally. A run using the closest local 6.1 source build (6.1.0-alpha.56) at the ticket's reported Isaac Lab revision stopped on an unrelated 4096-environment RTX allocation/OOM failure before reaching the reported invalid-slice path, so it is not counted as a reproduction. - The focused empty-annotator-frame regression test deterministically fails on the PR parent with the reported invalid-slice behavior and passes on this PR. The unbounded zero-agent commands were wrapped only in a timeout, and `OMNI_KIT_ACCEPT_EULA=YES` was scoped to the processes after confirming an existing accepted-EULA marker. Their command arguments were otherwise unchanged. #### Focused and repository checks - `uv run --frozen --extra test python -m pytest source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py -q` — 200 passed - The new focused regression tests were verified to fail before the fixes and pass afterward. - `uv run --frozen isaaclab -f` - `uv run --frozen --extra test python tools/changelog/cli.py check develop` ## Type of change - Bug fix (non-breaking change which fixes an issue) ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable. ## Checklist Docker and GPU tests run on demand. Push the commits you want tested, then comment `run-ci` on the pull request. - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `uv run isaaclab -f` - [x] I have made corresponding changes to the documentation (changelog fragments; no public API change) - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package (do **not** edit `CHANGELOG.rst` or bump `extension.toml` — CI handles that) - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- .../kguo-deformable-render-graph.rst | 4 ++ .../isaaclab_newton/physics/newton_manager.py | 35 +++++++--- .../renderers/newton_warp_renderer.py | 9 ++- .../test_newton_manager_abstraction.py | 66 +++++++++++++++++++ .../changelog.d/kguo-empty-rtx-frame.rst | 4 ++ .../renderers/isaac_rtx_renderer.py | 19 +++--- .../test_isaac_rtx_renderer_contract.py | 36 ++++++++++ 7 files changed, 154 insertions(+), 19 deletions(-) create mode 100644 source/isaaclab_newton/changelog.d/kguo-deformable-render-graph.rst create mode 100644 source/isaaclab_physx/changelog.d/kguo-empty-rtx-frame.rst diff --git a/source/isaaclab_newton/changelog.d/kguo-deformable-render-graph.rst b/source/isaaclab_newton/changelog.d/kguo-deformable-render-graph.rst new file mode 100644 index 000000000000..18e920b0e0df --- /dev/null +++ b/source/isaaclab_newton/changelog.d/kguo-deformable-render-graph.rst @@ -0,0 +1,4 @@ +Fixed +^^^^^ + +* Avoided conditional CUDA graph capture for Newton Warp rendering of deformable triangle meshes. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index f6749291c2cd..40b21c72b882 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -477,6 +477,7 @@ class NewtonManager(PhysicsManager): # Newton scene-query scheduling and graph execution. _sensor_tasks: dict[str, Callable[[], None]] = {} + _sensor_eager_tasks: set[str] = set() _sensor_graph: wp.Graph | None = None _sensor_flags: wp.array | None = None _sensor_flags_host: np.ndarray | None = None @@ -1167,6 +1168,7 @@ def clear(cls): NewtonManager._graph = None NewtonManager._graph_capture_pending = False NewtonManager._sensor_tasks = {} + NewtonManager._sensor_eager_tasks = set() NewtonManager._invalidate_sensor_graph() NewtonManager._sensor_state = None NewtonManager._sensor_state_dirty = True @@ -2693,12 +2695,13 @@ def get_contacts(cls) -> Contacts | None: return cls._contacts @classmethod - def _register_sensor_task(cls, name: str, update_fn: Callable[[], None]) -> None: - """Register a graph-capturable scene-query task. + def _register_sensor_task(cls, name: str, update_fn: Callable[[], None], *, graph_capturable: bool = True) -> None: + """Register a scene-query task. Args: name: Unique task name. - update_fn: Graph-capturable callable run by :meth:`_update_sensor_tasks`. + update_fn: Callable run by :meth:`_update_sensor_tasks`. + graph_capturable: Whether ``update_fn`` supports conditional CUDA graph capture. """ if name in cls._sensor_tasks: raise ValueError(f"Newton sensor task '{name}' is already registered.") @@ -2711,6 +2714,8 @@ def _register_sensor_task(cls, name: str, update_fn: Callable[[], None]) -> None if model.particle_count > 0 and model.bvh_particles is None: model.bvh_build_particles(state) cls._sensor_tasks[name] = update_fn + if not graph_capturable: + cls._sensor_eager_tasks.add(name) cls._sensor_state = state cls._sensor_state_dirty = True cls._invalidate_sensor_graph() @@ -2719,6 +2724,7 @@ def _register_sensor_task(cls, name: str, update_fn: Callable[[], None]) -> None def _unregister_sensor_task(cls, name: str) -> None: """Remove a scene-query task, ignoring unknown names.""" if cls._sensor_tasks.pop(name, None) is not None: + cls._sensor_eager_tasks.discard(name) cls._invalidate_sensor_graph() @classmethod @@ -2734,6 +2740,13 @@ def _update_sensor_tasks(cls, *names: str) -> None: cls._sensor_state = state cls._sensor_state_dirty = True cls._invalidate_sensor_graph() + if cls._sensor_eager_tasks.intersection(names): + if cls._sensor_state_dirty: + cls._refit_sensor_bvh() + cls._sensor_state_dirty = False + for name in names: + cls._sensor_tasks[name]() + return cfg = PhysicsManager._cfg use_cuda_graph = bool(getattr(cfg, "use_cuda_graph", False)) and "cuda" in str(PhysicsManager._device) if use_cuda_graph and cls._sensor_graph is None and not cls._sensor_graph_capture_failed: @@ -2750,7 +2763,7 @@ def _update_sensor_tasks(cls, *names: str) -> None: assert cls._sensor_flags is not None cls._sensor_flags_host.fill(0) cls._sensor_flags_host[0] = int(cls._sensor_state_dirty) - task_names = tuple(cls._sensor_tasks) + task_names = tuple(name for name in cls._sensor_tasks if name not in cls._sensor_eager_tasks) for name in names: cls._sensor_flags_host[1 + task_names.index(name)] = 1 cls._sensor_flags.assign(cls._sensor_flags_host) @@ -2806,19 +2819,21 @@ def _invalidate_sensor_graph(cls) -> None: @classmethod def _capture_sensor_graph(cls) -> None: """Capture BVH refit and scene-query tasks into a conditional graph.""" + graph_tasks = tuple( + update_fn for name, update_fn in cls._sensor_tasks.items() if name not in cls._sensor_eager_tasks + ) with wp.ScopedDevice(PhysicsManager._device): cls._refit_sensor_bvh() - for update_fn in cls._sensor_tasks.values(): + for update_fn in graph_tasks: update_fn() - cls._sensor_flags = wp.zeros(1 + len(cls._sensor_tasks), dtype=wp.int32, device=PhysicsManager._device) - cls._sensor_flags_host = np.zeros(1 + len(cls._sensor_tasks), dtype=np.int32) - update_fns = tuple(cls._sensor_tasks.values()) + cls._sensor_flags = wp.zeros(1 + len(graph_tasks), dtype=wp.int32, device=PhysicsManager._device) + cls._sensor_flags_host = np.zeros(1 + len(graph_tasks), dtype=np.int32) def pipeline() -> None: assert cls._sensor_flags is not None wp.capture_if(cls._sensor_flags[0:1], cls._refit_sensor_bvh) - for index, update_fn in enumerate(update_fns): + for index, update_fn in enumerate(graph_tasks): wp.capture_if(cls._sensor_flags[index + 1 : index + 2], update_fn) device = PhysicsManager._device @@ -2838,7 +2853,7 @@ def pipeline() -> None: cls._sensor_graph_capture_failed = True logger.warning("Newton sensor graph capture failed; falling back to eager execution.") else: - logger.info("Captured Newton sensor graph with %d task(s).", len(cls._sensor_tasks)) + logger.info("Captured Newton sensor graph with %d task(s).", len(graph_tasks)) @classmethod def get_num_envs(cls) -> int: diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index 1f6abd2f4863..03723ec47599 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -553,7 +553,14 @@ def render(self, render_data: RenderData): if render_data.sensor_task_name is None: render_data.sensor_task_name = f"newton_warp_render:{id(render_data)}" - NewtonManager._register_sensor_task(render_data.sensor_task_name, lambda: self._launch_render(render_data)) + tri_indices = self._newton_model.tri_indices + # Warp mesh refits allocate graph nodes and are not supported inside a conditional graph body. + graph_capturable = tri_indices is None or tri_indices.shape[0] == 0 + NewtonManager._register_sensor_task( + render_data.sensor_task_name, + lambda: self._launch_render(render_data), + graph_capturable=graph_capturable, + ) NewtonManager._update_sensor_tasks(render_data.sensor_task_name) # Post-render PPISP: HDR scene-linear → LDR RGBA. Source/destination diff --git a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py index 2462cb5fca0e..7ec052c84a83 100644 --- a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py +++ b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py @@ -350,6 +350,72 @@ def get_state(cls): assert status["rendered"] +def test_non_graph_capturable_sensor_task_runs_eagerly(monkeypatch): + """Sensor tasks with allocation-backed work should not attempt CUDA graph capture.""" + state = object() + model = SimpleNamespace(shape_count=0, particle_count=0, bvh_shapes=None, bvh_particles=None) + calls: list[str] = [] + + monkeypatch.setattr(NewtonManager, "get_model", classmethod(lambda cls: model)) + monkeypatch.setattr(NewtonManager, "get_state_0", classmethod(lambda cls: state)) + monkeypatch.setattr(NewtonManager, "get_state", classmethod(lambda cls: state)) + monkeypatch.setattr(NewtonManager, "_model", model, raising=False) + monkeypatch.setattr(NewtonManager, "_sensor_tasks", {}, raising=False) + monkeypatch.setattr(NewtonManager, "_sensor_eager_tasks", set(), raising=False) + monkeypatch.setattr(NewtonManager, "_sensor_state", None, raising=False) + monkeypatch.setattr(NewtonManager, "_sensor_state_dirty", True, raising=False) + monkeypatch.setattr(NewtonManager, "_sensor_graph", None, raising=False) + monkeypatch.setattr(NewtonManager, "_sensor_flags", None, raising=False) + monkeypatch.setattr(NewtonManager, "_sensor_flags_host", None, raising=False) + monkeypatch.setattr(NewtonManager, "_sensor_graph_capture_failed", False, raising=False) + monkeypatch.setattr(PhysicsManager, "_cfg", SimpleNamespace(use_cuda_graph=True), raising=False) + monkeypatch.setattr(PhysicsManager, "_device", "cuda:0", raising=False) + monkeypatch.setattr( + NewtonManager, + "_capture_sensor_graph", + classmethod(lambda cls: pytest.fail("Non-graph-capturable task attempted CUDA graph capture.")), + ) + + NewtonManager._register_sensor_task("render", lambda: calls.append("render"), graph_capturable=False) + NewtonManager._update_sensor_tasks("render") + + assert calls == ["render"] + assert NewtonManager._sensor_graph is None + assert NewtonManager._sensor_graph_capture_failed is False + + +@pytest.mark.parametrize( + ("triangle_count", "expected_graph_capturable"), + [ + pytest.param(None, True, id="no-triangle-array"), + pytest.param(0, True, id="empty-triangle-array"), + pytest.param(1, False, id="deformable-triangle-mesh"), + ], +) +def test_newton_warp_renderer_marks_triangle_mesh_refit_as_eager( + monkeypatch, triangle_count, expected_graph_capturable +): + """Deformable triangle-mesh rendering should opt out of conditional CUDA graph capture.""" + from isaaclab_newton.renderers.newton_warp_renderer import NewtonWarpRenderer + + registration: dict[str, object] = {} + + def register_task(cls, name, update_fn, *, graph_capturable=True): + registration.update(name=name, update_fn=update_fn, graph_capturable=graph_capturable) + + monkeypatch.setattr(NewtonManager, "_register_sensor_task", classmethod(register_task)) + monkeypatch.setattr(NewtonManager, "_update_sensor_tasks", classmethod(lambda cls, *names: None)) + + tri_indices = None if triangle_count is None else SimpleNamespace(shape=(triangle_count, 3)) + renderer = object.__new__(NewtonWarpRenderer) + renderer._newton_model = SimpleNamespace(tri_indices=tri_indices) + render_data = SimpleNamespace(sensor_task_name=None, ppisp_pipeline=None) + + renderer.render(render_data) + + assert registration["graph_capturable"] is expected_graph_capturable + + def test_sensor_bvh_shape_flags_are_fixed_before_builder_creation(monkeypatch): """Builder finalization includes collision-only shapes without a later BVH rebuild.""" import newton diff --git a/source/isaaclab_physx/changelog.d/kguo-empty-rtx-frame.rst b/source/isaaclab_physx/changelog.d/kguo-empty-rtx-frame.rst new file mode 100644 index 000000000000..ee44715caa3e --- /dev/null +++ b/source/isaaclab_physx/changelog.d/kguo-empty-rtx-frame.rst @@ -0,0 +1,4 @@ +Fixed +^^^^^ + +* Handled empty Isaac RTX annotator warm-up frames without invalid Warp slicing. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index a242c3c332ac..9c984e9f0378 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -585,6 +585,17 @@ def tiling_grid_shape(): else: tiled_data_buffer = output + # The RTX annotator may return an empty frame while its render product is warming up. + # Clear the destination so callers do not observe stale data, then wait for the next frame. + if data_type == str(RenderBufferKind.RGB_HDR) and data_type not in output_data: + assert render_data._hdr_scratch_wp is not None + buf_wp = render_data._hdr_scratch_wp + else: + buf_wp = output_data[data_type].warp + if tiled_data_buffer.size == 0: + buf_wp.zero_() + continue + # convert data buffer to warp array if isinstance(tiled_data_buffer, np.ndarray): # Let warp infer the dtype from numpy array instead of hardcoding uint8 @@ -619,14 +630,6 @@ def tiling_grid_shape(): if data_type == str(RenderBufferKind.RGB_HDR): tiled_data_buffer = tiled_data_buffer[:, :, :3].contiguous() - # The HDR annotator's destination is the user-visible ``output_data["rgb_hdr"]`` - # when they requested it explicitly; otherwise the renderer's internal - # scratch buffer that the PPISP pipeline reads. - if data_type == str(RenderBufferKind.RGB_HDR) and data_type not in output_data: - assert render_data._hdr_scratch_wp is not None - buf_wp = render_data._hdr_scratch_wp - else: - buf_wp = output_data[data_type].warp wp.launch( kernel=reshape_tiled_image, dim=(view_count, cfg.height, cfg.width), diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 589e7e54e393..4835274e4023 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -13,6 +13,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock, call, patch +import numpy as np import pytest import warp as wp from packaging import version @@ -433,6 +434,41 @@ def test_deterministic_flag_gates_rtx_determinism_settings(monkeypatch, stored, determinism_mock.assert_called_once_with(settings) +@pytest.mark.parametrize("data_type", ["rgba", "normals"]) +def test_render_treats_empty_annotator_frame_as_not_ready(monkeypatch, data_type): + """An empty warm-up frame should clear its output without slicing or launching a reshape.""" + _install_omni_stubs(monkeypatch) + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + annotator = MagicMock() + annotator.get_data.return_value = np.empty((0, 0, 4), dtype=np.uint8) + output_buffer = MagicMock() + render_data = SimpleNamespace( + annotators={data_type: annotator}, + output_data={data_type: SimpleNamespace(warp=output_buffer)}, + spec=SimpleNamespace( + view_count=1, + device="cpu", + cfg=SimpleNamespace(width=64, height=64), + ), + renderer_info={}, + ppisp_pipeline=None, + _hdr_scratch_wp=None, + ) + renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg() + + with ( + patch.object(rtx_renderer, "ensure_isaac_rtx_render_update"), + patch.object(rtx_renderer.wp, "launch") as launch, + ): + renderer.render(render_data) + + output_buffer.zero_.assert_called_once_with() + launch.assert_not_called() + + def test_isaac_rtx_read_output_clears_stale_metadata_and_keeps_seeded_keys(monkeypatch): """read_output replaces (not merges): a dropped annotator info resets its info entry, seeded keys persist.""" _install_omni_stubs(monkeypatch) From 7c6b4d7eb69dc7d196fabb7f1fb9fd7e157dc641 Mon Sep 17 00:00:00 2001 From: Antoine RICHARD Date: Mon, 7 Sep 2026 20:57:42 +0200 Subject: [PATCH 006/128] [Odin] Fix preset-based agent selection for rsl_rl, rl_games and sb3 (#7532) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Preset-based `--agent` auto-selection is dead code for `rsl_rl`, `rl_games` and `sb3`. Two defects sit **in series** on the same code path: 1. **The selection guard cannot see past the CLI default.** `_auto_select_agent` in `source/isaaclab_tasks/isaaclab_tasks/utils/preset_cli.py` was only reached when `args.agent is None`. `add_common_train_args` registers `--agent` with `default=agent_default`, and every backend except `skrl` passes a non-`None` `agent_default` (`rsl_rl_cfg_entry_point`, `rl_games_cfg_entry_point`, `sb3_cfg_entry_point`). The parsed value is therefore never `None`, so the branch never runs. 2. **The benchmark entrypoints never asked for it.** `benchmark_train_{rsl_rl,rl_games,sb3}.py` and `benchmark_play_{rsl_rl,rl_games,sb3}.py` called `setup_preset_cli(parser, argv)` without `agent_library`, so auto-selection was not attempted at all (and the registered-agent help listing was missing too). Only the `skrl` benchmark entrypoints wired it. ## Provenance and symptom Benchmark sweep dispatch `20260901-153531` (image built from `release/3.0.0` at `f88dbc59c82`, `rsl_rl`) produced **60 failed rows**: `resnet18` (30) and `theia_tiny` (30), failing 100% across all three renderers and all physics backends. Every one died before the first training step: ``` ValueError: Observation 'critic' in observation set 'critic' not found in the observations from the environment. Available observations from the environment: ['policy'] ``` Call chain: `OnPolicyRunner.__init__` → `ppo.py construct_algorithm` → `rsl_rl/utils/utils.py:233 resolve_obs_groups`. ## Why this is an agent-entrypoint bug, not a missing observation group `Isaac-Cartpole-Camera` already declares the correct pairing: ```python "agent_preset_compatibility": { "rsl_rl_cfg_entry_point": _RAW_CAMERA_PRESETS, "rsl_rl_feature_cfg_entry_point": ("resnet18", "theia_tiny"), ... } ``` and `CartpoleCameraFeaturePPORunnerCfg` sets `obs_groups` with `critic: ["policy"]`. The feature entry point exists, is registered, is correct — it was simply unreachable, so `resnet18`/`theia_tiny` ran against the raw-camera `CartpoleCameraPPORunnerCfg`, whose `obs_groups` asks for a `critic` group the env does not expose. Nothing needs to be added to the environment. ## Why both defects are in one PR Neither half fixes the observed failure alone; this was measured, not assumed. Reverting either half of the change and running the regression test: | state | result | | --- | --- | | defect 2 fixed only (`agent_library` wired, guard unchanged) | 4 failed — `--agent` still parses to the non-`None` default, so the guard rejects | | defect 1 fixed only (guard fixed, `agent_library` not wired) | 4 failed — `if agent_library and ...` is `False`, auto-selection never attempted | | both fixed | passes | They are two links in one chain, so splitting them yields a PR that fixes nothing observable and a PR that cannot be tested end-to-end. ## Scope: train **and** play, benchmark **and** non-benchmark To pre-empt the obvious question — this is not a play-only or a train-only fix. | path | affected by | fixed by | proven by | | --- | --- | --- | --- | | `benchmark_train_{rsl_rl,rl_games,sb3}` | defects 1 + 2 | `preset_cli.py` + `agent_library=` wiring | `test_training_request_selects_preset_compatible_agent` — **this is the observed 60-row failure** | | `benchmark_play_{rsl_rl,rl_games,sb3}` | defects 1 + 2 | `preset_cli.py` + `agent_library=` wiring | `test_play_request_selects_preset_compatible_agent` | | `isaaclab_rl` `train_*`/`play_*` (non-benchmark) | defect 1 only — they already pass `agent_library` | `preset_cli.py` alone; **no file in this PR touches them** | `test_setup_preset_cli_auto_selects_agent_over_non_none_default` | Each half of the benchmark wiring was reverted independently and re-tested: * revert the three **train** wirings → the 4 training selection cases fail. Load-bearing for the reported failure. * revert the three **play** wirings → the 2 playback selection cases fail. There is **no train/play asymmetry** that would justify wiring only one of them: both register `--agent` with the same non-`None` default via the same helper, both call `setup_preset_cli`, and both feed `args.agent` into the same `resolve_task_config(...)` and then into the same `OnPolicyRunner(...)` construction. The reason the sweep only surfaced the training failure is ordering, not asymmetry — a row that dies at `train_rc=1` never reaches playback. Measured on the playback path before the wiring was added: ``` $ BenchmarkPlayRequest(backend="rsl_rl", task="Isaac-Cartpole-Camera", presets=("resnet18",)) args.agent : rsl_rl_cfg_entry_point # raw-camera config, wrong # with the wiring: args.agent : rsl_rl_feature_cfg_entry_point # correct ``` Playback would therefore have loaded a feature-trained checkpoint into the raw-camera policy architecture. ## Fix chosen Detect an explicitly typed `--agent` by re-parsing the same argv into a namespace pre-seeded with a sentinel: argparse only applies a default for a destination the namespace does not already carry, so the sentinel survives unless the user actually typed the flag. Auto-selection runs only when it does survive; an explicit `--agent` still wins. The six benchmark entrypoints now pass `agent_library`. The probe uses the same parser on the same argv as the real parse, so its verdict *is* argparse's verdict. Verified across every spelling — `--agent V`, `--agent=V`, the abbreviation `--age V`, repeated flags, and `--agent` after a `--` separator (correctly not explicit: argparse does not set it in the real parse either). A literal argv scan for `--agent` would get the abbreviation wrong and silently override a user's explicit choice, which is why the probe is preferred; the repo's existing `ExplicitAction` idiom would work too but requires touching all ten `--agent` registration sites across three packages, and a missed site fails the same silent way. ### Rejected: make `--agent` default to `None` everywhere This is the obvious fix and it is not safe. `skrl` can default to `None` because `train_skrl.py` reconstructs the entry point from `--algorithm` when it is `None`. The other three pass `args_cli.agent` straight into `resolve_task_config(...)`, which has no such fallback — `hydra.py:619` sets `agent_cfg = load_cfg_from_registry(...) if agent_entry else None`, and the entrypoints then dereference `agent_cfg.max_iterations`. A `None` default would break every plain `--task=X` run that relies on the canonical entry point, i.e. the overwhelmingly common case. Fixing that would mean adding a fallback to each of the six benchmark entrypoints plus the four train/play entrypoints: a much wider blast radius than the bug. ### Blast radius Behavior changes only where auto-selection actually fires, and only when the user did **not** type `--agent`: * **Rule 1 (preset-based)** fires only for tasks that opted in with `agent_preset_compatibility` — today `Isaac-Cartpole-Camera` and the two cartpole-showcase tasks. That is the declared contract finally being honored. Anyone who wants the previous (broken) pairing can still pass `--agent rsl_rl_cfg_entry_point` explicitly; covered by a test. * **Rule 2 (default-absent)** fires only when `_cfg_entry_point` is *not* registered and exactly one other entry point is. That path previously resolved an unregistered entry point and crashed, so this is strictly a repair. * Everything else — no preset pairing declared, or the canonical default is registered — keeps the exact default it had; covered by a test for all three libraries. * `skrl` is unaffected: it already passed `agent_default=None`, and the explicit/implicit distinction collapses to the old `is None` check for it. `rl_games` and `sb3` share the defect and are fixed by the same change. **`sb3` has no task declaring `agent_preset_compatibility`, so there is no positive selection test for it** — its wiring is covered by the shared code path and by a default-preservation test only. ## Type of change - Bug fix (non-breaking change which fixes an issue) ## Screenshots Not applicable. ## Validation Regression tests were confirmed to fail on `develop` without the fix and pass with it. **Before the fix** (production changes reverted to `develop`, tests in place): ``` $ uv run --frozen --extra test python -m pytest source/isaaclab_tasks/test/core/test_preset_cli.py \ source/isaaclab/test/benchmark/test_api.py -q -p no:warnings \ -k "preset_compatible_agent or keeps_backend_default_agent or auto_selects_agent_over_non_none or keeps_explicit_agent" FAILED test_preset_cli.py::test_setup_preset_cli_auto_selects_agent_over_non_none_default FAILED test_api.py::test_training_request_selects_preset_compatible_agent[resnet18-rsl_rl] FAILED test_api.py::test_training_request_selects_preset_compatible_agent[resnet18-rl_games] FAILED test_api.py::test_training_request_selects_preset_compatible_agent[theia_tiny-rsl_rl] FAILED test_api.py::test_training_request_selects_preset_compatible_agent[theia_tiny-rl_games] 5 failed, 6 passed, 53 deselected E AssertionError: assert 'rl_games_cfg_entry_point' == 'rl_games_feature_cfg_entry_point' ``` **After the fix:** ``` $ uv run --frozen --extra test python -m pytest source/isaaclab_tasks/test/core/test_preset_cli.py \ source/isaaclab_tasks/test/core/test_hydra.py source/isaaclab/test/benchmark \ source/isaaclab_rl/test/test_entrypoints_common.py -q -p no:warnings 492 passed, 1 skipped in 30.89s $ uv run --frozen --extra test --extra skrl --extra sb3 --extra rl-games --extra rsl-rl \ python -m pytest source/isaaclab_rl/test/test_typed_preset_cli_train_play.py -q -p no:warnings 8 passed in 33.27s ``` (The last suite fails on a bare `--extra test` environment on `develop` too — `ModuleNotFoundError: No module named 'skrl'` etc. — so it was rerun with the RL extras.) ``` $ uv run --frozen python tools/changelog/cli.py check develop ✓ All modified packages have valid changelog fragments. $ uv run --frozen isaaclab -f all hooks passed ``` Tests added: * `source/isaaclab_tasks/test/core/test_preset_cli.py` — auto-selection over a non-`None` default; explicit `--agent` wins over the preset, parametrized over the three spellings argparse accepts. The two pre-existing auto-selection tests only covered `skrl` with `agent_default=None`, which is exactly why this bug went unnoticed. * `source/isaaclab/test/benchmark/test_api.py` — drives the real benchmark entrypoints via `BenchmarkTrainingRequest`/`BenchmarkPlayRequest` and `_parse_args`, asserting the feature entry point is selected for `resnet18`/`theia_tiny` × `rsl_rl`/`rl_games` on the train path (mirroring the failing rows), the same on the play path, and that the canonical default survives otherwise (`rsl_rl`, `rl_games`, `sb3`). Both assert the *selected agent config*, not merely absence of an exception. Not run: a real training job on the failing rows (requires GPU sim). The failure is a construction-time config selection, fully reproduced at CLI level. ## Related PRs — checked, no overlap * **#7491** (Select pretrained checkpoints from resolved task configs) touches `isaaclab_rl/entrypoints/common.py`, the four `play_*` entrypoints, the four `benchmark_play_*` entrypoints, `cartpole/__init__.py` and `test_preset_cli.py` — adjacent, but changes neither `agent_default` nor the selection guard. Textual conflicts are possible in `test_preset_cli.py` and the `benchmark_play_*` files (one-line `setup_preset_cli(...)` call); no semantic conflict. * **#6440** (`fix(rsl_rl): default obs_groups for new runners`) adds an `obs_groups` default in `isaaclab_rl/rsl_rl/utils.py`. It would mask this symptom for runners that omit `obs_groups`, but `CartpoleCameraPPORunnerCfg` sets `obs_groups` explicitly, so it does not fix these 60 rows — and it would not make the correct feature config get selected either. Complementary, not duplicate. * `gh pr list --search "agent preset"` returned nothing touching this code. ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` --------- Co-authored-by: hujc --- .../fix-benchmark-agent-library-wiring.rst | 7 +++ .../rl_games/benchmark_play_rl_games.py | 2 +- .../rl_games/benchmark_train_rl_games.py | 2 +- .../backends/rsl_rl/benchmark_play_rsl_rl.py | 2 +- .../backends/rsl_rl/benchmark_train_rsl_rl.py | 2 +- .../backends/sb3/benchmark_play_sb3.py | 2 +- .../backends/sb3/benchmark_train_sb3.py | 2 +- source/isaaclab/test/benchmark/test_api.py | 55 +++++++++++++++++++ .../fix-preset-agent-auto-selection.rst | 8 +++ .../isaaclab_tasks/utils/preset_cli.py | 41 ++++++++++++-- .../test/core/test_preset_cli.py | 48 ++++++++++++++++ 11 files changed, 159 insertions(+), 12 deletions(-) create mode 100644 source/isaaclab/changelog.d/fix-benchmark-agent-library-wiring.rst create mode 100644 source/isaaclab_tasks/changelog.d/fix-preset-agent-auto-selection.rst diff --git a/source/isaaclab/changelog.d/fix-benchmark-agent-library-wiring.rst b/source/isaaclab/changelog.d/fix-benchmark-agent-library-wiring.rst new file mode 100644 index 000000000000..0152650f6112 --- /dev/null +++ b/source/isaaclab/changelog.d/fix-benchmark-agent-library-wiring.rst @@ -0,0 +1,7 @@ +Fixed +^^^^^ + +* Fixed the ``rsl_rl``, ``rl_games`` and ``sb3`` benchmark train and play entrypoints not passing + ``agent_library`` to :func:`~isaaclab_tasks.utils.setup_preset_cli`, which disabled preset-based + ``--agent`` selection and the registered-agent help listing for those backends. Only the ``skrl`` + entrypoints wired it. diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_play_rl_games.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_play_rl_games.py index e8c295b61bcf..bc2e32631f29 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_play_rl_games.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_play_rl_games.py @@ -85,7 +85,7 @@ def _parse_args(argv: list[str]): ) add_launcher_args(parser) - args_cli, remaining_args = setup_preset_cli(parser, argv) + args_cli, remaining_args = setup_preset_cli(parser, argv, agent_library="rl_games") _common.enable_cameras_for_video(args_cli) sys.argv = [sys.argv[0]] + remaining_args diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_train_rl_games.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_train_rl_games.py index b6dd72dd4ad0..2a081851e0a8 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_train_rl_games.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_train_rl_games.py @@ -104,7 +104,7 @@ def _parse_args(argv: list[str]): add_success_cli_args(parser) - args_cli, remaining_args = setup_preset_cli(parser, argv) + args_cli, remaining_args = setup_preset_cli(parser, argv, agent_library="rl_games") validate_distributed_args(parser, args_cli) enable_cameras_for_video(args_cli) sys.argv = [sys.argv[0]] + remaining_args diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_play_rsl_rl.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_play_rsl_rl.py index 52e64262abb4..5345ff11b3d8 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_play_rsl_rl.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_play_rsl_rl.py @@ -86,7 +86,7 @@ def _parse_args(argv: list[str]) -> tuple[argparse.Namespace, list[str]]: ) add_launcher_args(parser) - args, remaining = setup_preset_cli(parser, argv) + args, remaining = setup_preset_cli(parser, argv, agent_library="rsl_rl") _common.enable_cameras_for_video(args) sys.argv = [sys.argv[0]] + remaining return args, remaining diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_train_rsl_rl.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_train_rsl_rl.py index cec0ce323d98..684eaf079b4d 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_train_rsl_rl.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_train_rsl_rl.py @@ -97,7 +97,7 @@ def _parse_args(argv: list[str]): add_success_cli_args(parser) - args_cli, remaining_args = setup_preset_cli(parser, argv) + args_cli, remaining_args = setup_preset_cli(parser, argv, agent_library="rsl_rl") validate_distributed_args(parser, args_cli) enable_cameras_for_video(args_cli) sys.argv = [sys.argv[0]] + remaining_args diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_play_sb3.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_play_sb3.py index 7f3aa1e6f171..fc64b3c136a9 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_play_sb3.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_play_sb3.py @@ -92,7 +92,7 @@ def _parse_args(argv: list[str]): ) add_launcher_args(parser) - args_cli, remaining_args = setup_preset_cli(parser, argv) + args_cli, remaining_args = setup_preset_cli(parser, argv, agent_library="sb3") _common.enable_cameras_for_video(args_cli) sys.argv = [sys.argv[0]] + remaining_args diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_train_sb3.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_train_sb3.py index a1a1a80d14cb..33a66014445c 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_train_sb3.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_train_sb3.py @@ -163,7 +163,7 @@ def _parse_args(argv: list[str]): add_success_cli_args(parser, include_check_success=False) add_launcher_args(parser) - args_cli, remaining_args = setup_preset_cli(parser, argv) + args_cli, remaining_args = setup_preset_cli(parser, argv, agent_library="sb3") enable_cameras_for_video(args_cli) sys.argv = [sys.argv[0]] + remaining_args diff --git a/source/isaaclab/test/benchmark/test_api.py b/source/isaaclab/test/benchmark/test_api.py index 0cf850d966ee..2832d7fd57ac 100644 --- a/source/isaaclab/test/benchmark/test_api.py +++ b/source/isaaclab/test/benchmark/test_api.py @@ -155,6 +155,61 @@ def test_play_request_uses_backend_arguments(backend: str, monkeypatch) -> None: assert remaining_args == [] +@pytest.mark.parametrize("backend", ["rsl_rl", "rl_games"]) +@pytest.mark.parametrize("preset", ["resnet18", "theia_tiny"]) +def test_training_request_selects_preset_compatible_agent(backend: str, preset: str, monkeypatch) -> None: + """A feature preset picks the matching agent entry point instead of the backend default. + + ``Isaac-Cartpole-Camera`` declares ``resnet18``/``theia_tiny`` as compatible + only with its ``*_feature_cfg_entry_point``. Running the raw-camera default + against those presets builds a runner whose observation groups do not exist. + """ + import isaaclab_tasks # noqa: F401 + + request = BenchmarkTrainingRequest(backend=backend, task="Isaac-Cartpole-Camera", presets=(preset,)) + argv = dispatch._request_argv(request) + monkeypatch.setattr(sys, "argv", ["benchmark", *argv]) + entrypoint = importlib.import_module(dispatch._workflow_module("training", backend)) + # RSL-RL's adapter returns an extra CLI-helper module alongside (args, remaining). + args = entrypoint._parse_args(argv)[0] + + assert args.agent == f"{backend}_feature_cfg_entry_point" + + +@pytest.mark.parametrize("backend", ["rsl_rl", "rl_games"]) +def test_play_request_selects_preset_compatible_agent(backend: str, monkeypatch) -> None: + """Playback resolves the same agent config, so it needs the same preset pairing. + + A benchmark sweep only reaches playback once training succeeds, so this path + mis-selects the raw-camera entry point in exactly the same way, and would + then load a feature-trained checkpoint into the wrong policy architecture. + """ + import isaaclab_tasks # noqa: F401 + + request = BenchmarkPlayRequest(backend=backend, task="Isaac-Cartpole-Camera", presets=("resnet18",)) + argv = dispatch._request_argv(request) + monkeypatch.setattr(sys, "argv", ["benchmark", *argv]) + entrypoint = importlib.import_module(dispatch._workflow_module("play", backend)) + args = entrypoint._parse_args(argv)[0] + + assert args.agent == f"{backend}_feature_cfg_entry_point" + + +@pytest.mark.parametrize("backend", ["rsl_rl", "rl_games", "sb3"]) +def test_training_request_keeps_backend_default_agent_without_presets(backend: str, monkeypatch) -> None: + """Without a preset pairing the backend's canonical default entry point stands.""" + import isaaclab_tasks # noqa: F401 + + request = BenchmarkTrainingRequest(backend=backend, task="Isaac-Cartpole", max_iterations=1) + argv = dispatch._request_argv(request) + monkeypatch.setattr(sys, "argv", ["benchmark", *argv]) + entrypoint = importlib.import_module(dispatch._workflow_module("training", backend)) + # RSL-RL's adapter returns an extra CLI-helper module alongside (args, remaining). + args = entrypoint._parse_args(argv)[0] + + assert args.agent == f"{backend}_cfg_entry_point" + + @pytest.mark.parametrize("configured_output_dir", [None, "/tmp/custom-videos"]) def test_play_backend_configures_video_before_environment_creation( monkeypatch, tmp_path, configured_output_dir diff --git a/source/isaaclab_tasks/changelog.d/fix-preset-agent-auto-selection.rst b/source/isaaclab_tasks/changelog.d/fix-preset-agent-auto-selection.rst new file mode 100644 index 000000000000..636eea3b20f8 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/fix-preset-agent-auto-selection.rst @@ -0,0 +1,8 @@ +Fixed +^^^^^ + +* Fixed preset-based ``--agent`` auto-selection being skipped for every entrypoint that registers + ``--agent`` with a non-``None`` default (``rsl_rl``, ``rl_games`` and ``sb3``). The selection guard + could not tell a default-supplied value from a user-typed one, so ``presets=resnet18`` and + ``presets=theia_tiny`` on ``Isaac-Cartpole-Camera`` kept the raw-camera entry point and the runner + failed to construct. An explicitly typed ``--agent`` still wins over auto-selection. diff --git a/source/isaaclab_tasks/isaaclab_tasks/utils/preset_cli.py b/source/isaaclab_tasks/isaaclab_tasks/utils/preset_cli.py index cd7f1abbb94b..e709f44bc1e3 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/utils/preset_cli.py +++ b/source/isaaclab_tasks/isaaclab_tasks/utils/preset_cli.py @@ -92,7 +92,8 @@ def setup_preset_cli( triggers ``--help`` rendering. agent_library: Optional RL-library prefix. When provided, task-specific help lists registered ``--agent`` values and declared preset - compatibility. + compatibility, and ``args.agent`` is auto-selected from the active + presets unless the user typed ``--agent`` explicitly. Returns: ``(args, remaining)`` where ``remaining`` is the verbatim output of @@ -136,7 +137,7 @@ def setup_preset_cli( args, remaining = parser.parse_known_args(args_to_parse) task_name = getattr(args, "task", None) or argv_helper.task_name - if agent_library and getattr(args, "agent", None) is None and task_name: + if agent_library and task_name and hasattr(args, "agent") and not _agent_passed_explicitly(parser, args_to_parse): _auto_select_agent(args, task_name, agent_library, args_to_parse) return args, remaining @@ -313,6 +314,35 @@ def build(agent_library: str, task_name: str | None) -> str: # ============================================================================ +_AGENT_UNSET = object() +"""Sentinel seeded onto a probe namespace to detect a user-typed ``--agent``.""" + + +def _agent_passed_explicitly(parser: argparse.ArgumentParser, argv: list[str]) -> bool: + """Return whether *argv* carries a user-typed ``--agent`` value. + + Entry-point parsers register ``--agent`` with a non-``None`` default (e.g. + ``rsl_rl_cfg_entry_point``), so the parsed value alone cannot tell an + explicit choice from a default-supplied one. Re-parsing into a namespace + pre-seeded with a sentinel answers that: argparse only applies a default for + a destination the namespace does not already carry, so the sentinel survives + unless the user actually typed the flag. Delegating to argparse keeps + abbreviations (``--age``) and ``--agent=VALUE`` handled the same way the real + parse handles them. + + Args: + parser: Parser that already parsed *argv* successfully. + argv: Argument list handed to ``parse_known_args``. + + Returns: + ``True`` when the user typed ``--agent``, ``False`` when the parsed value + came from the argument's default. + """ + probe = argparse.Namespace(agent=_AGENT_UNSET) + parser.parse_known_args(argv, namespace=probe) + return probe.agent is not _AGENT_UNSET + + def _auto_select_agent( args: argparse.Namespace, task_name: str, @@ -333,7 +363,9 @@ def _auto_select_agent( This handles tasks such as ``IsaacContrib-Humanoid-AMP-*`` that only support a non-default algorithm (AMP) and never register the PPO default. - Does nothing when the match is absent or ambiguous. + Leaves ``args.agent`` untouched when the match is absent or ambiguous, so + the caller's default stands. Callers must skip this when the user typed + ``--agent`` explicitly; see :func:`_agent_passed_explicitly`. Args: args: Parsed namespace to update in-place. @@ -341,9 +373,6 @@ def _auto_select_agent( agent_library: RL-library prefix (e.g. ``"skrl"``). argv: Raw argument list scanned for ``presets=`` tokens. """ - if getattr(args, "agent", None) is not None: - return - active_presets: set[str] = set() for token in argv: if token.startswith("presets="): diff --git a/source/isaaclab_tasks/test/core/test_preset_cli.py b/source/isaaclab_tasks/test/core/test_preset_cli.py index 1820f70185b8..78796694eb66 100644 --- a/source/isaaclab_tasks/test/core/test_preset_cli.py +++ b/source/isaaclab_tasks/test/core/test_preset_cli.py @@ -535,3 +535,51 @@ def test_setup_preset_cli_auto_selects_agent_when_default_absent(monkeypatch): argv = ["--task", "IsaacContrib-Humanoid-AMP-Walk-Direct"] args, _ = setup_preset_cli(parser, argv, agent_library="skrl") assert args.agent == "skrl_amp_cfg_entry_point" + + +def _make_agent_parser(default: str) -> argparse.ArgumentParser: + """Mimic an entrypoint parser that gives ``--agent`` a non-``None`` default.""" + parser = _make_parser() + parser.add_argument("--agent", type=str, default=default) + return parser + + +def test_setup_preset_cli_auto_selects_agent_over_non_none_default(monkeypatch): + """A feature preset overrides the entrypoint's canonical ``--agent`` default. + + ``rsl_rl``/``rl_games``/``sb3`` entrypoints register ``--agent`` with a + non-``None`` default, so auto-selection has to distinguish that default from + a user-typed value. Without that distinction ``presets=resnet18`` keeps the + raw-camera entry point and the runner fails to construct. + """ + import isaaclab_tasks # noqa: F401 + + monkeypatch.setattr("sys.argv", ["train.py", "--task", "Isaac-Cartpole-Camera"]) + from isaaclab_tasks.utils.preset_cli import setup_preset_cli + + parser = _make_agent_parser("rsl_rl_cfg_entry_point") + argv = ["--task", "Isaac-Cartpole-Camera", "presets=resnet18"] + args, _ = setup_preset_cli(parser, argv, agent_library="rsl_rl") + assert args.agent == "rsl_rl_feature_cfg_entry_point" + + +@pytest.mark.parametrize( + "agent_tokens", + [["--agent", "rsl_rl_cfg_entry_point"], ["--agent=rsl_rl_cfg_entry_point"], ["--age", "rsl_rl_cfg_entry_point"]], + ids=["separate", "equals", "abbreviated"], +) +def test_setup_preset_cli_keeps_explicit_agent_over_preset(monkeypatch, agent_tokens): + """An explicitly typed ``--agent`` wins over preset-based auto-selection. + + Covers every spelling argparse accepts for the flag, since explicitness is + determined by re-parsing rather than by scanning argv for a literal token. + """ + import isaaclab_tasks # noqa: F401 + + monkeypatch.setattr("sys.argv", ["train.py", "--task", "Isaac-Cartpole-Camera"]) + from isaaclab_tasks.utils.preset_cli import setup_preset_cli + + parser = _make_agent_parser("rsl_rl_cfg_entry_point") + argv = ["--task", "Isaac-Cartpole-Camera", *agent_tokens, "presets=resnet18"] + args, _ = setup_preset_cli(parser, argv, agent_library="rsl_rl") + assert args.agent == "rsl_rl_cfg_entry_point" From 4cf35ccc4fd81bd6c955a5cd090d41296b7d2b92 Mon Sep 17 00:00:00 2001 From: Antoine RICHARD Date: Mon, 7 Sep 2026 21:09:05 +0200 Subject: [PATCH 007/128] Fix compute_first_contact/air missing transitions as the sensor clock ages (#7574) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description `ContactSensor.compute_first_contact(dt)` / `compute_first_air(dt)` silently missed most touchdowns and lift-offs once the simulation had run for a few seconds. On a transition step the contact (resp. air) timer is exactly one sensor update interval, so the check degenerated to `dt < dt + 1e-8`. The sensor clock is a float32 accumulator whose rounding error reaches ~1e-6 after a few seconds of simulated time, roughly 100x the default tolerance, so the events vanished depending on the magnitude of the sim clock. The reporter's CPU reproducer shows 352/500 touchdowns and lift-offs missed at the default tolerance; it reproduces exactly on `develop`. All three backends (Newton, PhysX, OVPhysX) share the same logic, and the 2.x torch sensor had the same defect, so this is long-standing rather than a Newton regression. **Fix.** `abs_tol` now defaults to `None` and is resolved at call time by a single helper in `BaseContactSensor` to **half the sensor update interval**, i.e. `0.5 * max(cfg.update_period, physics_dt)`. Valid timer values are integer multiples of that interval, so half an interval is the midpoint between "one update ago" and "two updates ago". This stays robust to float32 clock drift for hours of simulated time and works for both `history_length == 0` (lazy, once-per-policy-step refresh) and `history_length > 0` (every physics substep). The Warp kernels are unchanged. Passing an explicit `abs_tol` still works; callers who want the previous behaviour can pass `abs_tol=1e-8`. **Tests.** Public-API regression tests in the Newton, PhysX and OVPhysX contact sensor suites: settle a body in contact, age the sensor clock to 2.5 / 10 / 30 s, lift and land the body, and poll `compute_first_contact` / `compute_first_air` with default arguments. These fail on the unfixed code (e.g. `compute_first_air missed the lift-off at clock 2.833s: reported [], expected [0]`) and pass with the fix. The Newton variant covers both `history_length` modes. **Refresh before query.** Both methods now call the sensor's outdated-buffer refresh before comparing, restoring what the 2.x torch sensor did by reading through `self.data`. The Warp port (#4707 / #4716) read the private timer buffer directly, so a `history_length == 0` caller that polled before touching `data` saw the previous step's timers. The aged-clock tests now poll before reading `data`; the Newton lazy-mode cases fail without the refresh (transition reported one policy step late) and pass with it. Adopted from the approach in #7294. **Out of scope, noted for follow-up.** - The transition interval is counted in both the ending and the starting phase, so `last_air_time` / `last_contact_time` overestimate each phase by one update interval (also raised in the issue). That bias is present since 2.x and baked into tuned `feet_air_time` thresholds, so changing it would rescale rewards for trained velocity policies and deserves its own PR. - On `cuda:0` with `history_length == 0`, the PhysX and OVPhysX sensors serve stale net forces when buffers refresh only on data access, independently of this change. That is why the PhysX/OVPhysX public-API tests cover only the substep cadence. Fixes #7283 ## Type of change - Bug fix (non-breaking change which fixes an issue) ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package (do **not** edit `CHANGELOG.rst` or bump `extension.toml` — CI handles that) - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- ...antoiner-first-contact-tolerance.minor.rst | 13 +++ .../contact_sensor/base_contact_sensor.py | 36 +++++-- ...antoiner-first-contact-tolerance.minor.rst | 12 +++ .../sensors/contact_sensor/contact_sensor.py | 26 +++-- .../test/sensors/test_contact_sensor.py | 101 ++++++++++++++++++ ...antoiner-first-contact-tolerance.minor.rst | 12 +++ .../sensors/contact_sensor/contact_sensor.py | 24 +++-- .../test/sensors/test_contact_sensor.py | 98 +++++++++++++++++ ...antoiner-first-contact-tolerance.minor.rst | 12 +++ .../sensors/contact_sensor/contact_sensor.py | 26 +++-- .../test/sensors/test_contact_sensor.py | 98 +++++++++++++++++ 11 files changed, 430 insertions(+), 28 deletions(-) create mode 100644 source/isaaclab/changelog.d/antoiner-first-contact-tolerance.minor.rst create mode 100644 source/isaaclab_newton/changelog.d/antoiner-first-contact-tolerance.minor.rst create mode 100644 source/isaaclab_ov/changelog.d/antoiner-first-contact-tolerance.minor.rst create mode 100644 source/isaaclab_physx/changelog.d/antoiner-first-contact-tolerance.minor.rst diff --git a/source/isaaclab/changelog.d/antoiner-first-contact-tolerance.minor.rst b/source/isaaclab/changelog.d/antoiner-first-contact-tolerance.minor.rst new file mode 100644 index 000000000000..910d7023a8c6 --- /dev/null +++ b/source/isaaclab/changelog.d/antoiner-first-contact-tolerance.minor.rst @@ -0,0 +1,13 @@ +Fixed +^^^^^ + +* Fixed :meth:`~isaaclab.sensors.contact_sensor.BaseContactSensor.compute_first_contact` and + :meth:`~isaaclab.sensors.contact_sensor.BaseContactSensor.compute_first_air` silently missing + touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). Their + ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update interval + instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 rounding + error of the sensor clock, so most transitions were dropped. Callers that relied on the previous + behavior can pass ``abs_tol=1e-8`` explicitly. + Both methods now also refresh outdated sensor buffers before comparing, so a sensor with + ``history_length=0`` no longer reports the previous step's transitions when it is queried before + its data is read. diff --git a/source/isaaclab/isaaclab/sensors/contact_sensor/base_contact_sensor.py b/source/isaaclab/isaaclab/sensors/contact_sensor/base_contact_sensor.py index d520251bc59e..34000f62f7f6 100644 --- a/source/isaaclab/isaaclab/sensors/contact_sensor/base_contact_sensor.py +++ b/source/isaaclab/isaaclab/sensors/contact_sensor/base_contact_sensor.py @@ -148,13 +148,34 @@ def find_sensors(self, name_keys: str | Sequence[str], preserve_order: bool = Fa """ return string_utils.resolve_matching_names(name_keys, self.body_names, preserve_order) + def _resolve_first_transition_tolerance(self, abs_tol: float | None) -> float: + """Resolves the tolerance used to detect a first contact or first air transition. + + Valid air and contact timers are integer multiples of the sensor update interval, so half an + interval is the midpoint between "one update ago" and "two updates ago". Using it as the + tolerance keeps the comparison robust to the float32 rounding error of the sensor clock, + which grows with simulated time and quickly exceeds any fixed tolerance. + + Args: + abs_tol: The caller-provided tolerance [s]. If None, half the sensor update interval + is used. + + Returns: + The absolute tolerance to add to the queried time period [s]. + """ + if abs_tol is not None: + return abs_tol + # An update period of 0.0 means the sensor is updated on every physics step. + return 0.5 * max(self.cfg.update_period, self._sim_physics_dt) + @abstractmethod - def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: + def compute_first_contact(self, dt: float, abs_tol: float | None = None) -> ProxyArray: """Checks if bodies that have established contact within the last :attr:`dt` seconds. This function checks if the bodies have established contact within the last :attr:`dt` seconds by comparing the current contact time with the given time period. If the contact time is less - than the given time period, then the bodies are considered to be in contact. + than the given time period, then the bodies are considered to be in contact. Outdated sensor + buffers are refreshed before the comparison. .. note:: The function assumes that :attr:`dt` is a factor of the sensor update time-step. In other @@ -164,7 +185,8 @@ def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArra Args: dt: The time period since the contact was established. - abs_tol: The absolute tolerance for the comparison. + abs_tol: The absolute tolerance for the comparison [s]. Defaults to None, in which case + half the sensor update interval is used. Returns: A boolean tensor indicating the bodies that have established contact within the last @@ -177,12 +199,13 @@ def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArra raise NotImplementedError(f"Compute first contact is not implemented for {self.__class__.__name__}.") @abstractmethod - def compute_first_air(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: + def compute_first_air(self, dt: float, abs_tol: float | None = None) -> ProxyArray: """Checks if bodies that have broken contact within the last :attr:`dt` seconds. This function checks if the bodies have broken contact within the last :attr:`dt` seconds by comparing the current air time with the given time period. If the air time is less - than the given time period, then the bodies are considered to not be in contact. + than the given time period, then the bodies are considered to not be in contact. Outdated sensor + buffers are refreshed before the comparison. .. note:: It assumes that :attr:`dt` is a factor of the sensor update time-step. In other words, @@ -192,7 +215,8 @@ def compute_first_air(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: Args: dt: The time period since the contract is broken. - abs_tol: The absolute tolerance for the comparison. + abs_tol: The absolute tolerance for the comparison [s]. Defaults to None, in which case + half the sensor update interval is used. Returns: A boolean tensor indicating the bodies that have broken contact within the last :attr:`dt` seconds. diff --git a/source/isaaclab_newton/changelog.d/antoiner-first-contact-tolerance.minor.rst b/source/isaaclab_newton/changelog.d/antoiner-first-contact-tolerance.minor.rst new file mode 100644 index 000000000000..f7d9db23c442 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/antoiner-first-contact-tolerance.minor.rst @@ -0,0 +1,12 @@ +Fixed +^^^^^ + +* Fixed :meth:`compute_first_contact` and :meth:`compute_first_air` on the contact sensor silently + missing touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). + Their ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update + interval instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 + rounding error of the sensor clock, so most transitions were dropped. Callers that relied on the + previous behavior can pass ``abs_tol=1e-8`` explicitly. + Both methods now also refresh outdated sensor buffers before comparing, so a sensor with + ``history_length=0`` no longer reports the previous step's transitions when it is queried before + its data is read. diff --git a/source/isaaclab_newton/isaaclab_newton/sensors/contact_sensor/contact_sensor.py b/source/isaaclab_newton/isaaclab_newton/sensors/contact_sensor/contact_sensor.py index 4a020b9dccc5..b4cf8b0b18f8 100644 --- a/source/isaaclab_newton/isaaclab_newton/sensors/contact_sensor/contact_sensor.py +++ b/source/isaaclab_newton/isaaclab_newton/sensors/contact_sensor/contact_sensor.py @@ -200,12 +200,13 @@ def find_sensors(self, name_keys: str | Sequence[str], preserve_order: bool = Fa ) return string_utils.resolve_matching_names(name_keys, sensor_names, preserve_order) - def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: + def compute_first_contact(self, dt: float, abs_tol: float | None = None) -> ProxyArray: """Checks if sensors that have established contact within the last :attr:`dt` seconds. This function checks if the sensors have established contact within the last :attr:`dt` seconds by comparing the current contact time with the given time period. If the contact time is less - than the given time period, then the sensors are considered to be in contact. + than the given time period, then the sensors are considered to be in contact. Outdated sensor + buffers are refreshed before the comparison. Note: The function assumes that :attr:`dt` is a factor of the sensor update time-step. In other @@ -215,7 +216,8 @@ def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArra Args: dt: The time period since the contact was established. - abs_tol: The absolute tolerance for the comparison. + abs_tol: The absolute tolerance for the comparison [s]. Defaults to None, in which case + half the sensor update interval is used. Returns: A float array (1.0/0.0) indicating the sensors that have established contact within the @@ -232,21 +234,25 @@ def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArra "The contact sensor is not configured to track contact time." "Please enable the 'track_air_time' in the sensor configuration." ) + tol = self._resolve_first_transition_tolerance(abs_tol) + # refresh lazily updated buffers so the timers reflect the current physics step + self._update_outdated_buffers() wp.launch( compute_first_transition_kernel, dim=(self._num_envs, self._num_sensors), - inputs=[float(dt + abs_tol), self._data._current_contact_time], + inputs=[float(dt + tol), self._data._current_contact_time], outputs=[self._data._first_transition], device=self._device, ) return self._data._first_transition_ta - def compute_first_air(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: + def compute_first_air(self, dt: float, abs_tol: float | None = None) -> ProxyArray: """Checks if sensors that have broken contact within the last :attr:`dt` seconds. This function checks if the sensors have broken contact within the last :attr:`dt` seconds by comparing the current air time with the given time period. If the air time is less - than the given time period, then the sensors are considered to not be in contact. + than the given time period, then the sensors are considered to not be in contact. Outdated sensor + buffers are refreshed before the comparison. Note: It assumes that :attr:`dt` is a factor of the sensor update time-step. In other words, @@ -256,7 +262,8 @@ def compute_first_air(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: Args: dt: The time period since the contract is broken. - abs_tol: The absolute tolerance for the comparison. + abs_tol: The absolute tolerance for the comparison [s]. Defaults to None, in which case + half the sensor update interval is used. Returns: A float array (1.0/0.0) indicating the sensors that have broken contact within the last @@ -274,10 +281,13 @@ def compute_first_air(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: "Please enable the 'track_air_time' in the sensor configuration." ) + tol = self._resolve_first_transition_tolerance(abs_tol) + # refresh lazily updated buffers so the timers reflect the current physics step + self._update_outdated_buffers() wp.launch( compute_first_transition_kernel, dim=(self._num_envs, self._num_sensors), - inputs=[float(dt + abs_tol), self._data._current_air_time], + inputs=[float(dt + tol), self._data._current_air_time], outputs=[self._data._first_transition], device=self._device, ) diff --git a/source/isaaclab_newton/test/sensors/test_contact_sensor.py b/source/isaaclab_newton/test/sensors/test_contact_sensor.py index 34245c2d906e..c299c77589d2 100644 --- a/source/isaaclab_newton/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_newton/test/sensors/test_contact_sensor.py @@ -1194,3 +1194,104 @@ def test_invalid_expression_raises_regex_error(): """Reject malformed selector expressions at contact sensor construction.""" with pytest.raises(re.error): _compile_label_pattern("foo(") + + +@pytest.mark.parametrize("device", test_devices()) +@pytest.mark.parametrize("clock_age", [2.5, 10.0, 30.0]) +@pytest.mark.parametrize("history_length", [1, 0], ids=["substep_refresh", "lazy_refresh"]) +def test_first_transition_with_aged_clock(device: str, clock_age: float, history_length: int): + """Regression for #7283: transitions must still be reported once the sensor clock has aged. + + The sensor clock is a float32 accumulator whose rounding error grows with simulated time. On a + transition step the contact (resp. air) timer is exactly one polling period, so the default + tolerance of :meth:`compute_first_contact` has to absorb that error. A fixed 1e-8 tolerance is + ~100x too small after a few seconds and silently drops touchdowns and lift-offs. + """ + # With history, the sensor refreshes every physics step; without it, only when data is read. + decimation = 1 if history_length > 0 else 4 + poll_dt = decimation * SIM_DT + settle_steps = 40 + poll_steps = 120 // decimation + + sim_cfg = make_sim_cfg(use_mujoco_contacts=False, device=device, gravity=(0.0, 0.0, -9.81)) + with build_simulation_context(sim_cfg=sim_cfg, auto_add_lighting=True) as sim: + sim._app_control_on_stop_handle = None + + scene_cfg = ContactSensorTestSceneCfg(num_envs=1, env_spacing=5.0) + scene_cfg.object_a = create_shape_cfg( + ShapeType.BOX, + "{ENV_REGEX_NS}/Object", + pos=(0.0, 0.0, get_shape_height(ShapeType.BOX) / 2), + disable_gravity=False, + activate_contact_sensors=True, + ) + scene_cfg.contact_sensor_a = ContactSensorCfg( + prim_path="{ENV_REGEX_NS}/Object", + update_period=0.0, + history_length=history_length, + track_air_time=True, + ) + + scene = InteractiveScene(scene_cfg) + sim.reset() + scene.reset() + + sensor: ContactSensor = scene["contact_sensor_a"] + obj: RigidObject = scene["object_a"] + + def _in_contact() -> bool: + """Ground truth for the contact state, read through the public data accessor.""" + return torch.norm(sensor.data.net_normal_forces_w.torch, dim=-1).max().item() > 0.1 + + # Let the box come to rest on the ground so the sensor starts in contact. + for _ in range(settle_steps): + perform_sim_step(sim, scene, SIM_DT) + assert _in_contact(), "Box should be resting on the ground before the clock is aged." + + # Age the sensor clock without stepping physics: the resting contact state is unchanged, so + # this isolates the float32 clock drift from any change in the contact forces. + for tick in range(int(round(clock_age / SIM_DT))): + sensor.update(SIM_DT) + if history_length == 0 and (tick + 1) % decimation == 0: + _in_contact() # lazy refresh, mirroring a policy-rate reader + aged_clock = wp.to_torch(sensor._timestamp).max().item() + assert aged_clock == pytest.approx(clock_age + settle_steps * SIM_DT, abs=0.05) + + # Launch the box so that it leaves the ground and lands again within the polling window. + velocity = torch.zeros(1, 6, device=obj.device) + velocity[:, 2] = 3.0 + obj.write_root_velocity_to_sim_index(root_velocity=velocity) + + reported_air: list[int] = [] + reported_contact: list[int] = [] + expected_air: list[int] = [] + expected_contact: list[int] = [] + was_in_contact = True + for step in range(poll_steps): + for _ in range(decimation): + perform_sim_step(sim, scene, SIM_DT) + # Poll before anything reads ``data`` this step: the query itself must refresh lazily + # updated buffers, otherwise it reports the previous step's timers. + first_contact = sensor.compute_first_contact(poll_dt).torch.any().item() + first_air = sensor.compute_first_air(poll_dt).torch.any().item() + in_contact = _in_contact() + if in_contact and not was_in_contact: + expected_contact.append(step) + if not in_contact and was_in_contact: + expected_air.append(step) + was_in_contact = in_contact + if first_contact: + reported_contact.append(step) + if first_air: + reported_air.append(step) + + assert len(expected_air) == 1, f"Expected exactly one lift-off in the window; got {expected_air}." + assert len(expected_contact) == 1, f"Expected exactly one touchdown in the window; got {expected_contact}." + assert reported_air == expected_air, ( + f"compute_first_air missed or mis-reported the lift-off at clock {aged_clock:.3f}s: " + f"reported {reported_air}, expected {expected_air}." + ) + assert reported_contact == expected_contact, ( + f"compute_first_contact missed or mis-reported the touchdown at clock {aged_clock:.3f}s: " + f"reported {reported_contact}, expected {expected_contact}." + ) diff --git a/source/isaaclab_ov/changelog.d/antoiner-first-contact-tolerance.minor.rst b/source/isaaclab_ov/changelog.d/antoiner-first-contact-tolerance.minor.rst new file mode 100644 index 000000000000..f7d9db23c442 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/antoiner-first-contact-tolerance.minor.rst @@ -0,0 +1,12 @@ +Fixed +^^^^^ + +* Fixed :meth:`compute_first_contact` and :meth:`compute_first_air` on the contact sensor silently + missing touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). + Their ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update + interval instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 + rounding error of the sensor clock, so most transitions were dropped. Callers that relied on the + previous behavior can pass ``abs_tol=1e-8`` explicitly. + Both methods now also refresh outdated sensor buffers before comparing, so a sensor with + ``history_length=0`` no longer reports the previous step's transitions when it is queried before + its data is read. diff --git a/source/isaaclab_ov/isaaclab_ov/sensors/contact_sensor/contact_sensor.py b/source/isaaclab_ov/isaaclab_ov/sensors/contact_sensor/contact_sensor.py index f92689d09a43..f8b40b0f40f1 100644 --- a/source/isaaclab_ov/isaaclab_ov/sensors/contact_sensor/contact_sensor.py +++ b/source/isaaclab_ov/isaaclab_ov/sensors/contact_sensor/contact_sensor.py @@ -423,12 +423,15 @@ def reset(self, env_ids: Sequence[int] | None = None, env_mask: wp.array | None device=self._device, ) - def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: + def compute_first_contact(self, dt: float, abs_tol: float | None = None) -> ProxyArray: """Boolean mask (as float) of bodies that established contact within ``dt`` [s]. + Outdated sensor buffers are refreshed before the comparison. + Args: dt: Time window since contact establishment [s]. - abs_tol: Absolute tolerance for the comparison [s]. + abs_tol: Absolute tolerance for the comparison [s]. Defaults to None, in which case + half the sensor update interval is used. Returns: Boolean tensor (1.0/0.0) of shape ``(num_envs, num_sensors)``. @@ -441,21 +444,27 @@ def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArra "The contact sensor is not configured to track contact time." " Please enable 'track_air_time' in the sensor configuration." ) + tol = self._resolve_first_transition_tolerance(abs_tol) + # refresh lazily updated buffers so the timers reflect the current physics step + self._update_outdated_buffers() wp.launch( compute_first_transition_kernel, dim=(self._num_envs, self._num_sensors), - inputs=[float(dt + abs_tol), self._data._current_contact_time], + inputs=[float(dt + tol), self._data._current_contact_time], outputs=[self._data._first_transition], device=self._device, ) return self._data._first_transition_ta - def compute_first_air(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: + def compute_first_air(self, dt: float, abs_tol: float | None = None) -> ProxyArray: """Boolean mask (as float) of bodies that broke contact within ``dt`` [s]. + Outdated sensor buffers are refreshed before the comparison. + Args: dt: Time window since contact break [s]. - abs_tol: Absolute tolerance for the comparison [s]. + abs_tol: Absolute tolerance for the comparison [s]. Defaults to None, in which case + half the sensor update interval is used. Returns: Boolean tensor (1.0/0.0) of shape ``(num_envs, num_sensors)``. @@ -468,10 +477,13 @@ def compute_first_air(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: "The contact sensor is not configured to track air time." " Please enable 'track_air_time' in the sensor configuration." ) + tol = self._resolve_first_transition_tolerance(abs_tol) + # refresh lazily updated buffers so the timers reflect the current physics step + self._update_outdated_buffers() wp.launch( compute_first_transition_kernel, dim=(self._num_envs, self._num_sensors), - inputs=[float(dt + abs_tol), self._data._current_air_time], + inputs=[float(dt + tol), self._data._current_air_time], outputs=[self._data._first_transition], device=self._device, ) diff --git a/source/isaaclab_ov/test/sensors/test_contact_sensor.py b/source/isaaclab_ov/test/sensors/test_contact_sensor.py index 87b46ff901f2..dd29a68a6c5e 100644 --- a/source/isaaclab_ov/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ov/test/sensors/test_contact_sensor.py @@ -336,6 +336,104 @@ def test_sphere_contact_time(device): _run_contact_sensor_test(SPHERE_CFG, _SIM_DT, device, _TERRAINS, _DURATIONS) +@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("clock_age", [2.5, 10.0, 30.0]) +def test_first_transition_with_aged_clock(device, clock_age): + """Regression for #7283: transitions must still be reported once the sensor clock has aged. + + The sensor clock is a float32 accumulator whose rounding error grows with simulated time. On a + transition step the contact (resp. air) timer is exactly one polling period, so the default + tolerance of :meth:`ContactSensor.compute_first_contact` has to absorb that error. A fixed 1e-8 + tolerance is orders of magnitude too small after a few seconds of simulated time. + """ + # The sensor keeps history, so it refreshes every physics step and is polled at that same rate. + # The lazy (zero-history) cadence is covered by the kernel-level tolerance test: on GPU this + # backend serves stale contact forces when buffers refresh only on data access, which is + # unrelated to the tolerance under test here. + history_length = 1 + decimation = 1 + poll_dt = decimation * _SIM_DT + poll_steps = 16 + + with _ovphysx_sim_context(device=device, dt=_SIM_DT, add_lighting=True) as sim: + scene_cfg = ContactSensorSceneCfg(num_envs=1, env_spacing=1.0) + scene_cfg.terrain = FLAT_TERRAIN_CFG + scene_cfg.shape = CUBE_CFG + scene_cfg.contact_sensor = ContactSensorCfg( + prim_path=CUBE_CFG.prim_path, + track_pose=True, + debug_vis=False, + update_period=0.0, + track_air_time=True, + history_length=history_length, + track_contact_points=False, + track_friction_forces=False, + filter_prim_paths_expr=[], + ) + scene = InteractiveScene(scene_cfg) + sim.reset() + + sensor: ContactSensor = scene["contact_sensor"] + shape: RigidObject = scene["shape"] + contact_pose = CUBE_CFG.contact_pose.to(device=shape.device).unsqueeze(0) + non_contact_pose = CUBE_CFG.non_contact_pose.to(device=shape.device).unsqueeze(0) + + def _in_contact() -> bool: + """Ground truth for the contact state, read through the public data accessor.""" + return torch.norm(sensor.data.net_normal_forces_w.torch, dim=-1).max().item() > 0.1 + + def _hold(pose: torch.Tensor, num_steps: int) -> None: + """Pin the cube to a pose for the given number of physics steps.""" + for _ in range(num_steps): + shape.write_root_pose_to_sim_index(root_pose=pose) + _perform_sim_step(sim, scene, _SIM_DT) + + # Settle the cube on the ground so the sensor starts in contact. + _hold(contact_pose, 8) + assert _in_contact(), "Cube should be in contact with the ground before the clock is aged." + + # Age the sensor clock without stepping physics: the resting contact state is unchanged, so + # this isolates the float32 clock drift from any change in the contact forces. + for tick in range(int(round(clock_age / _SIM_DT))): + sensor.update(_SIM_DT) + aged_clock = wp.to_torch(sensor._timestamp).max().item() + assert aged_clock == pytest.approx(clock_age + 8 * _SIM_DT, abs=0.05) + + # Lift the cube off the ground for half the window, then set it back down. + reported_air: list[int] = [] + reported_contact: list[int] = [] + expected_air: list[int] = [] + expected_contact: list[int] = [] + was_in_contact = True + for step in range(poll_steps): + _hold(non_contact_pose if step < poll_steps // 2 else contact_pose, decimation) + # Poll before anything reads ``data`` this step: the query itself must refresh lazily + # updated buffers, otherwise it reports the previous step's timers. + first_contact = sensor.compute_first_contact(poll_dt).torch.any().item() + first_air = sensor.compute_first_air(poll_dt).torch.any().item() + in_contact = _in_contact() + if in_contact and not was_in_contact: + expected_contact.append(step) + if not in_contact and was_in_contact: + expected_air.append(step) + was_in_contact = in_contact + if first_contact: + reported_contact.append(step) + if first_air: + reported_air.append(step) + + assert len(expected_air) == 1, f"Expected exactly one lift-off in the window; got {expected_air}." + assert len(expected_contact) == 1, f"Expected exactly one touchdown in the window; got {expected_contact}." + assert reported_air == expected_air, ( + f"compute_first_air missed or mis-reported the lift-off at clock {aged_clock:.3f}s: " + f"reported {reported_air}, expected {expected_air}." + ) + assert reported_contact == expected_contact, ( + f"compute_first_contact missed or mis-reported the touchdown at clock {aged_clock:.3f}s: " + f"reported {reported_contact}, expected {expected_contact}." + ) + + @pytest.mark.parametrize("device", ["cuda:0", "cpu"]) @pytest.mark.parametrize("num_envs", [1, 6, 24]) def test_cube_stack_contact_filtering(device, num_envs): diff --git a/source/isaaclab_physx/changelog.d/antoiner-first-contact-tolerance.minor.rst b/source/isaaclab_physx/changelog.d/antoiner-first-contact-tolerance.minor.rst new file mode 100644 index 000000000000..f7d9db23c442 --- /dev/null +++ b/source/isaaclab_physx/changelog.d/antoiner-first-contact-tolerance.minor.rst @@ -0,0 +1,12 @@ +Fixed +^^^^^ + +* Fixed :meth:`compute_first_contact` and :meth:`compute_first_air` on the contact sensor silently + missing touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). + Their ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update + interval instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 + rounding error of the sensor clock, so most transitions were dropped. Callers that relied on the + previous behavior can pass ``abs_tol=1e-8`` explicitly. + Both methods now also refresh outdated sensor buffers before comparing, so a sensor with + ``history_length=0`` no longer reports the previous step's transitions when it is queried before + its data is read. diff --git a/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py b/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py index f39cdb1c46ce..f46bbde43c51 100644 --- a/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py +++ b/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py @@ -219,12 +219,13 @@ def reset(self, env_ids: Sequence[int] | None = None, env_mask: wp.array | None device=self._device, ) - def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: + def compute_first_contact(self, dt: float, abs_tol: float | None = None) -> ProxyArray: """Checks if bodies that have established contact within the last :attr:`dt` seconds. This function checks if the bodies have established contact within the last :attr:`dt` seconds by comparing the current contact time with the given time period. If the contact time is less - than the given time period, then the bodies are considered to be in contact. + than the given time period, then the bodies are considered to be in contact. Outdated sensor + buffers are refreshed before the comparison. .. note:: The function assumes that :attr:`dt` is a factor of the sensor update time-step. In other @@ -239,7 +240,8 @@ def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArra Args: dt: The time period since the contact was established. - abs_tol: The absolute tolerance for the comparison. + abs_tol: The absolute tolerance for the comparison [s]. Defaults to None, in which case + half the sensor update interval is used. Returns: A boolean tensor indicating the bodies that have established contact within the last @@ -255,21 +257,25 @@ def compute_first_contact(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArra "The contact sensor is not configured to track contact time." "Please enable the 'track_air_time' in the sensor configuration." ) + tol = self._resolve_first_transition_tolerance(abs_tol) + # refresh lazily updated buffers so the timers reflect the current physics step + self._update_outdated_buffers() wp.launch( compute_first_transition_kernel, dim=(self._num_envs, self._num_sensors), - inputs=[float(dt + abs_tol), self._data._current_contact_time], + inputs=[float(dt + tol), self._data._current_contact_time], outputs=[self._data._first_transition], device=self._device, ) return self._data._first_transition_ta - def compute_first_air(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: + def compute_first_air(self, dt: float, abs_tol: float | None = None) -> ProxyArray: """Checks if bodies that have broken contact within the last :attr:`dt` seconds. This function checks if the bodies have broken contact within the last :attr:`dt` seconds by comparing the current air time with the given time period. If the air time is less - than the given time period, then the bodies are considered to not be in contact. + than the given time period, then the bodies are considered to not be in contact. Outdated sensor + buffers are refreshed before the comparison. .. note:: It assumes that :attr:`dt` is a factor of the sensor update time-step. In other words, @@ -284,7 +290,8 @@ def compute_first_air(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: Args: dt: The time period since the contract is broken. - abs_tol: The absolute tolerance for the comparison. + abs_tol: The absolute tolerance for the comparison [s]. Defaults to None, in which case + half the sensor update interval is used. Returns: A boolean tensor indicating the bodies that have broken contact within the last :attr:`dt` seconds. @@ -300,10 +307,13 @@ def compute_first_air(self, dt: float, abs_tol: float = 1.0e-8) -> ProxyArray: "Please enable the 'track_air_time' in the sensor configuration." ) + tol = self._resolve_first_transition_tolerance(abs_tol) + # refresh lazily updated buffers so the timers reflect the current physics step + self._update_outdated_buffers() wp.launch( compute_first_transition_kernel, dim=(self._num_envs, self._num_sensors), - inputs=[float(dt + abs_tol), self._data._current_air_time], + inputs=[float(dt + tol), self._data._current_air_time], outputs=[self._data._first_transition], device=self._device, ) diff --git a/source/isaaclab_physx/test/sensors/test_contact_sensor.py b/source/isaaclab_physx/test/sensors/test_contact_sensor.py index e8a98d3360ee..6a9114da8524 100644 --- a/source/isaaclab_physx/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_physx/test/sensors/test_contact_sensor.py @@ -393,6 +393,104 @@ def test_sphere_contact_time(setup_simulation, disable_contact_processing): _run_contact_sensor_test(SPHERE_CFG, sim_dt, devices, terrains, settings, durations) +@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("clock_age", [2.5, 10.0, 30.0]) +def test_first_transition_with_aged_clock(setup_simulation, device, clock_age): + """Regression for #7283: transitions must still be reported once the sensor clock has aged. + + The sensor clock is a float32 accumulator whose rounding error grows with simulated time. On a + transition step the contact (resp. air) timer is exactly one polling period, so the default + tolerance of :meth:`ContactSensor.compute_first_contact` has to absorb that error. A fixed 1e-8 + tolerance is orders of magnitude too small after a few seconds of simulated time. + """ + sim_dt = setup_simulation[0] + # The sensor keeps history, so it refreshes every physics step and is polled at that same rate. + # The lazy (zero-history) cadence is covered by the kernel-level tolerance test: on GPU this + # backend serves stale contact forces when buffers refresh only on data access, which is + # unrelated to the tolerance under test here. + history_length = 1 + decimation = 1 + poll_dt = decimation * sim_dt + poll_steps = 16 + + with build_simulation_context(device=device, dt=sim_dt, add_lighting=True) as sim: + sim._app_control_on_stop_handle = None + + scene_cfg = ContactSensorSceneCfg(num_envs=1, env_spacing=1.0) + scene_cfg.terrain = FLAT_TERRAIN_CFG + scene_cfg.shape = CUBE_CFG + scene_cfg.contact_sensor = ContactSensorCfg( + prim_path=CUBE_CFG.prim_path, + track_pose=True, + debug_vis=False, + update_period=0.0, + track_air_time=True, + history_length=history_length, + ) + scene = InteractiveScene(scene_cfg) + sim.reset() + + sensor: ContactSensor = scene["contact_sensor"] + shape: RigidObject = scene["shape"] + contact_pose = CUBE_CFG.contact_pose.to(device=shape.device).unsqueeze(0) + non_contact_pose = CUBE_CFG.non_contact_pose.to(device=shape.device).unsqueeze(0) + + def _in_contact() -> bool: + """Ground truth for the contact state, read through the public data accessor.""" + return torch.norm(sensor.data.net_normal_forces_w.torch, dim=-1).max().item() > 0.1 + + def _hold(pose: torch.Tensor, num_steps: int) -> None: + """Pin the cube to a pose for the given number of physics steps.""" + for _ in range(num_steps): + shape.write_root_pose_to_sim_index(root_pose=pose) + _perform_sim_step(sim, scene, sim_dt) + + # Settle the cube on the ground so the sensor starts in contact. + _hold(contact_pose, 8) + assert _in_contact(), "Cube should be in contact with the ground before the clock is aged." + + # Age the sensor clock without stepping physics: the resting contact state is unchanged, so + # this isolates the float32 clock drift from any change in the contact forces. + for tick in range(int(round(clock_age / sim_dt))): + sensor.update(sim_dt) + aged_clock = wp.to_torch(sensor._timestamp).max().item() + assert aged_clock == pytest.approx(clock_age + 8 * sim_dt, abs=0.05) + + # Lift the cube off the ground for half the window, then set it back down. + reported_air: list[int] = [] + reported_contact: list[int] = [] + expected_air: list[int] = [] + expected_contact: list[int] = [] + was_in_contact = True + for step in range(poll_steps): + _hold(non_contact_pose if step < poll_steps // 2 else contact_pose, decimation) + # Poll before anything reads ``data`` this step: the query itself must refresh lazily + # updated buffers, otherwise it reports the previous step's timers. + first_contact = sensor.compute_first_contact(poll_dt).torch.any().item() + first_air = sensor.compute_first_air(poll_dt).torch.any().item() + in_contact = _in_contact() + if in_contact and not was_in_contact: + expected_contact.append(step) + if not in_contact and was_in_contact: + expected_air.append(step) + was_in_contact = in_contact + if first_contact: + reported_contact.append(step) + if first_air: + reported_air.append(step) + + assert len(expected_air) == 1, f"Expected exactly one lift-off in the window; got {expected_air}." + assert len(expected_contact) == 1, f"Expected exactly one touchdown in the window; got {expected_contact}." + assert reported_air == expected_air, ( + f"compute_first_air missed or mis-reported the lift-off at clock {aged_clock:.3f}s: " + f"reported {reported_air}, expected {expected_air}." + ) + assert reported_contact == expected_contact, ( + f"compute_first_contact missed or mis-reported the touchdown at clock {aged_clock:.3f}s: " + f"reported {reported_contact}, expected {expected_contact}." + ) + + @pytest.mark.parametrize("device", ["cuda:0", "cpu"]) @pytest.mark.parametrize("num_envs", [1, 6, 24]) def test_cube_stack_contact_filtering(setup_simulation, device, num_envs): From f948fa596c1c61a155d55778976b7547799e6200 Mon Sep 17 00:00:00 2001 From: Zeng Qingcheng <60593302+NeoZng@users.noreply.github.com> Date: Tue, 8 Sep 2026 04:25:16 +0900 Subject: [PATCH 008/128] Fix MJWarp USD friction loss import (#7298) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Fixes #6829. Preserve MuJoCo-authored joint friction loss when Isaac Lab imports USD stages for the Newton MJWarp backend. Before this change, both Newton production import paths passed only `SchemaResolverNewton` and `SchemaResolverPhysx` to Newton: * vectorized clone replication; and * standalone stage import. Consequently, a joint authored with `mjc:frictionloss=0.11` finalized with `Model.joint_friction=0.0`, even though Newton supports the attribute through `SchemaResolverMjc`. This PR makes USD resolver selection an active-manager policy: * `NewtonManager` defaults to Newton then PhysX resolvers; * `NewtonMJWarpManager` appends the MuJoCo resolver; * clone and standalone imports consume the same manager-owned resolver list; and * resolver order remains Newton → PhysX → MuJoCo, so MJC values are fallbacks and do not override higher-priority authored values. MuJoCo custom attributes continue to be registered through the existing `_builder_attribute_solvers = (SolverMuJoCo,)` mechanism. The follow-up removes the redundant MJWarp registration override instead of duplicating that base-class path. ## Scope and alternative considered PR #7386 fixes the immediate `frictionloss` symptom by appending the complete `SchemaResolverMjc` to every Newton physics import. That resolver also interprets additional joint, shape, contact, and scene attributes. Applying it unconditionally would therefore expand MJC semantics to Featherstone, XPBD, VBD, Kamino, and MPM rather than changing only MJWarp. This PR intentionally keeps the immediate fix solver-scoped: the manager that registers and consumes MuJoCo-specific attributes also owns the MuJoCo resolver. Regression coverage verifies that MJWarp imports `mjc:frictionloss` and `mjc:damping`, while Featherstone preserves its previous behavior. A broader cross-backend solution should separately classify portable MJC core semantics from MuJoCo-specific extensions, ideally by splitting those resolver responsibilities in Newton upstream. That larger architecture change is outside the scope of this bug fix. No dependency or public API is added. ## Type of change * Bug fix (non-breaking change which fixes an issue) ## Release backport * [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable; this fixes a non-visual USD import path. ## Verification Latest follow-up commit: * focused production-path regression matrix: 5 passed; * MJWarp × clone/standalone preserves `frictionloss=0.11` and `damping=0.23`; * Featherstone × clone/standalone leaves both MJC-authored values unconsumed; * the existing explicit-global clone import test still passes; * full pre-commit suite passed; * changelog validation passed; * Python bytecode compilation passed; and * `git diff --check` passed. Previous PR head verification in the matching isolated environment: * Python 3.12.13; * `isaaclab-newton==5.4.0`; * `newton==1.5.0`; * `warp-lang==1.16.0`; * `mujoco-warp==3.11.0`; and * full Newton manager abstraction suite: 157 passed on a CUDA host. ## Checklist * [x] I have read and understood the [[contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html)](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) * [x] I have run the pre-commit checks * [x] I have made corresponding changes to the documentation (not applicable: no public API or user workflow changed) * [x] My changes generate no new warnings * [x] I have added tests that prove my fix is effective * [x] I have added a changelog fragment under `source//changelog.d/` for every touched package * [x] I have added my name to `CONTRIBUTORS.md` --------- Co-authored-by: NeoZng --- .../isaaclab_contrib/coupling/coupler.py | 9 ++ .../test/coupling/test_coupler.py | 50 +++++++ .../test/custom_coupling/test_manager.py | 31 +++++ .../neozng-mjwarp-usd-joint-properties.rst | 5 + .../isaaclab_newton/cloner/replicate.py | 3 +- .../isaaclab_newton/physics/newton_manager.py | 24 +++- .../cloner/test_newton_builder_world_hook.py | 4 +- .../test_newton_manager_abstraction.py | 129 ++++++++++++++++++ 8 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 source/isaaclab_newton/changelog.d/neozng-mjwarp-usd-joint-properties.rst diff --git a/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py b/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py index 2310dc664223..ea290af0ec3e 100644 --- a/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py +++ b/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py @@ -23,6 +23,7 @@ from isaaclab_newton.physics.newton_manager import NewtonManager from isaaclab_newton.physics.vbd_manager import NewtonVBDManager from newton import CollisionPipeline, Model, ModelBuilder, ShapeFlags +from newton.solvers import SolverBase from newton.solvers.experimental.coupled import SolverCoupled, SolverCoupledADMM, SolverCoupledProxy from isaaclab.physics import PhysicsManager @@ -191,6 +192,14 @@ def _register_builder_attributes(cls, builder: ModelBuilder) -> None: for entry in PhysicsManager._cfg.solver_cfg.entries: entry.solver_cfg.class_type._register_builder_attributes(builder) + @classmethod + def _registers_builder_attributes_from_solver(cls, solver_cls: type[SolverBase]) -> bool: + """Return whether the manager or a configured nested entry registers ``solver_cls`` attributes.""" + return super()._registers_builder_attributes_from_solver(solver_cls) or any( + entry.solver_cfg.class_type._registers_builder_attributes_from_solver(solver_cls) + for entry in PhysicsManager._cfg.solver_cfg.entries + ) + @classmethod def _prepare_builder_for_finalize(cls, builder: ModelBuilder) -> None: """Normalize kinematic colliders when a coupled entry uses implicit MPM.""" diff --git a/source/isaaclab_contrib/test/coupling/test_coupler.py b/source/isaaclab_contrib/test/coupling/test_coupler.py index 131deb8b82f2..a13e2e99d942 100644 --- a/source/isaaclab_contrib/test/coupling/test_coupler.py +++ b/source/isaaclab_contrib/test/coupling/test_coupler.py @@ -17,6 +17,7 @@ from dataclasses import dataclass, field from types import SimpleNamespace +import isaaclab_newton.physics.newton_manager as newton_manager_module import numpy as np import pytest from isaaclab_newton.physics import ( @@ -24,6 +25,7 @@ KaminoPADMMSolverCfg, MJWarpSolverCfg, MPMSolverCfg, + NewtonCfg, NewtonCollisionPipelineCfg, NewtonVBDManager, VBDSolverCfg, @@ -33,6 +35,8 @@ from newton import ModelBuilder, ShapeFlags from newton.solvers.experimental.coupled import SolverCoupledADMM, SolverCoupledProxy +from pxr import Sdf, Usd, UsdGeom, UsdPhysics + from isaaclab_contrib.coupling import ( CouplerAdmmCfg, CouplerCfg, @@ -596,6 +600,52 @@ def test_nested_solvers_register_their_builder_attributes(monkeypatch): assert builder.has_custom_attribute("mpm:young_modulus") +@pytest.mark.parametrize( + ("entry_solver_cfg", "expected_friction", "expected_damping"), + [ + pytest.param(MJWarpSolverCfg(), 0.11, 0.23, id="mjwarp"), + pytest.param(XPBDSolverCfg(), 0.0, 0.0, id="xpbd"), + ], +) +def test_nested_solver_scopes_mujoco_joint_properties( + monkeypatch, entry_solver_cfg, expected_friction, expected_damping +): + """A coupler imports MuJoCo properties only when a nested solver consumes them.""" + solver_cfg = CouplerProxyCfg(entries=[CouplerEntryCfg(name="rigid", solver_cfg=entry_solver_cfg)]) + monkeypatch.setattr(coupler.PhysicsManager, "_cfg", NewtonCfg(solver_cfg=solver_cfg)) + + stage = Usd.Stage.CreateInMemory() + UsdGeom.Xform.Define(stage, "/World") + root_path = "/World/robot" + root = UsdGeom.Cube.Define(stage, root_path).GetPrim() + UsdPhysics.RigidBodyAPI.Apply(root) + UsdPhysics.ArticulationRootAPI.Apply(root) + child_path = f"{root_path}/child" + child = UsdGeom.Cube.Define(stage, child_path).GetPrim() + UsdPhysics.RigidBodyAPI.Apply(child) + joint = UsdPhysics.RevoluteJoint.Define(stage, f"{child_path}/joint") + joint.CreateAxisAttr().Set("Z") + joint.CreateBody0Rel().SetTargets([root_path]) + joint.CreateBody1Rel().SetTargets([child_path]) + joint.GetPrim().CreateAttribute("mjc:frictionloss", Sdf.ValueTypeNames.Double, True).Set(0.11) + joint.GetPrim().CreateAttribute("mjc:damping", Sdf.ValueTypeNames.Double, True).Set(0.23) + + monkeypatch.setattr(newton_manager_module, "get_current_stage", lambda: stage) + monkeypatch.setattr(newton_manager_module, "_restore_visible_colliders_without_visual_shapes", lambda *args: None) + monkeypatch.setattr(newton_manager_module, "replace_newton_builder_shape_colors", lambda *args: None) + monkeypatch.setattr(newton_manager_module, "import_builder_visual_material_paths", lambda *args: None) + monkeypatch.setattr(NewtonManager, "_builder", None) + monkeypatch.setattr(NewtonManager, "_deformable_registry", []) + monkeypatch.setattr(NewtonManager, "_per_world_builder_hooks", []) + + NewtonCouplerManager.instantiate_builder_from_stage() + builder = NewtonManager._builder + model = builder.finalize(device="cpu") + + assert model.joint_friction.numpy()[-1] == pytest.approx(expected_friction) + assert model.joint_damping.numpy()[-1] == pytest.approx(expected_damping) + + def test_contact_initialization_prepares_coupled_solver_buffers(monkeypatch): """Entry-local contact buffers are allocated before graph capture.""" events: list[tuple[str, object | None]] = [] diff --git a/source/isaaclab_contrib/test/custom_coupling/test_manager.py b/source/isaaclab_contrib/test/custom_coupling/test_manager.py index d05962e96437..cf09dd77d293 100644 --- a/source/isaaclab_contrib/test/custom_coupling/test_manager.py +++ b/source/isaaclab_contrib/test/custom_coupling/test_manager.py @@ -12,6 +12,8 @@ from isaaclab_newton.physics import MJWarpSolverCfg, VBDSolverCfg from newton import ModelBuilder +from pxr import Sdf, Usd, UsdGeom, UsdPhysics + import isaaclab_contrib.custom_coupling.coupled_mjwarp_vbd_manager as manager_module from isaaclab_contrib.custom_coupling.coupled_mjwarp_vbd_manager import NewtonCoupledMJWarpVBDManager from isaaclab_contrib.custom_coupling.newton_manager_cfg import CoupledMJWarpVBDSolverCfg @@ -28,6 +30,35 @@ def test_register_builder_attributes_includes_nested_solvers(monkeypatch: pytest assert builder.has_custom_attribute("mujoco:condim") +def test_registered_mujoco_solver_imports_mujoco_joint_properties(monkeypatch: pytest.MonkeyPatch) -> None: + """The coupled manager imports joint properties consumed by its MuJoCo solver.""" + cfg = CoupledMJWarpVBDSolverCfg() + monkeypatch.setattr(manager_module.PhysicsManager, "_cfg", SimpleNamespace(solver_cfg=cfg)) + + stage = Usd.Stage.CreateInMemory() + root_path = "/World/robot" + root = UsdGeom.Cube.Define(stage, root_path).GetPrim() + UsdPhysics.RigidBodyAPI.Apply(root) + UsdPhysics.ArticulationRootAPI.Apply(root) + child_path = f"{root_path}/child" + child = UsdGeom.Cube.Define(stage, child_path).GetPrim() + UsdPhysics.RigidBodyAPI.Apply(child) + joint = UsdPhysics.RevoluteJoint.Define(stage, f"{child_path}/joint") + joint.CreateAxisAttr().Set("Z") + joint.CreateBody0Rel().SetTargets([root_path]) + joint.CreateBody1Rel().SetTargets([child_path]) + joint.GetPrim().CreateAttribute("mjc:frictionloss", Sdf.ValueTypeNames.Double, True).Set(0.11) + joint.GetPrim().CreateAttribute("mjc:damping", Sdf.ValueTypeNames.Double, True).Set(0.23) + + builder = ModelBuilder() + NewtonCoupledMJWarpVBDManager._register_builder_attributes(builder) + builder.add_usd(stage, schema_resolvers=NewtonCoupledMJWarpVBDManager._get_usd_import_schema_resolvers()) + model = builder.finalize(device="cpu") + + assert model.joint_friction.numpy()[-1] == pytest.approx(0.11) + assert model.joint_damping.numpy()[-1] == pytest.approx(0.23) + + def test_reset_forwards_to_both_subsolvers(monkeypatch: pytest.MonkeyPatch) -> None: """Reset the real sub-solvers instead of the dummy solver slot.""" rigid_solver = MagicMock() diff --git a/source/isaaclab_newton/changelog.d/neozng-mjwarp-usd-joint-properties.rst b/source/isaaclab_newton/changelog.d/neozng-mjwarp-usd-joint-properties.rst new file mode 100644 index 000000000000..87714973c404 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/neozng-mjwarp-usd-joint-properties.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Fixed MuJoCo-based solver managers dropping ``mjc:frictionloss`` during USD + stage imports. diff --git a/source/isaaclab_newton/isaaclab_newton/cloner/replicate.py b/source/isaaclab_newton/isaaclab_newton/cloner/replicate.py index 06bd4aee18cc..b7366ee3fe1d 100644 --- a/source/isaaclab_newton/isaaclab_newton/cloner/replicate.py +++ b/source/isaaclab_newton/isaaclab_newton/cloner/replicate.py @@ -14,7 +14,6 @@ import numpy as np import warp as wp from newton import ModelBuilder -from newton._src.usd.schemas import SchemaResolverNewton, SchemaResolverPhysx from pxr import Usd @@ -110,8 +109,8 @@ def _build_newton_builder_from_mapping( quaternions = np.zeros((mapping.shape[1], 4), dtype=np.float32) quaternions[:, 3] = 1.0 - schema_resolvers = [SchemaResolverNewton(), SchemaResolverPhysx()] manager_cls = PhysicsManager._sim.physics_manager + schema_resolvers = manager_cls._get_usd_import_schema_resolvers() builder = manager_cls.create_builder(up_axis=up_axis) import_paths = (PhysicsManager._sim.cfg.physics_prim_path, *global_paths) diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index 40b21c72b882..180707943815 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -75,8 +75,8 @@ def _paused_gc(): from newton.sensors import SensorContact as NewtonContactSensor from newton.sensors import SensorFrameTransform from newton.sensors import SensorIMU as NewtonSensorIMU -from newton.solvers import SolverBase, SolverKamino -from newton.usd import SchemaResolverNewton, SchemaResolverPhysx +from newton.solvers import SolverBase, SolverKamino, SolverMuJoCo +from newton.usd import SchemaResolver, SchemaResolverMjc, SchemaResolverNewton, SchemaResolverPhysx from pxr import Usd, UsdGeom @@ -1259,6 +1259,11 @@ def _register_builder_attributes(cls, builder: ModelBuilder) -> None: for solver_cls in cls._builder_attribute_solvers: solver_cls.register_custom_attributes(builder) + @classmethod + def _registers_builder_attributes_from_solver(cls, solver_cls: type[SolverBase]) -> bool: + """Return whether this manager registers custom attributes from ``solver_cls``.""" + return solver_cls in cls._builder_attribute_solvers + @classmethod def _prepare_builder_for_finalize(cls, builder: ModelBuilder) -> None: """Subclass hook to normalize *builder* before model finalization. @@ -1916,6 +1921,19 @@ def _get_usd_import_ignore_paths(cls) -> list[str]: """Return solver-specific prim paths excluded from USD import.""" return [] + @classmethod + def _get_usd_import_schema_resolvers(cls) -> list[SchemaResolver]: + """Return ordered schema resolvers for physics-model USD imports. + + MJC is enabled for managers that register ``SolverMuJoCo`` attributes. + Visualization and articulation-ordering builders keep their fixed pair + because solver attributes do not affect their outputs. + """ + resolvers: list[SchemaResolver] = [SchemaResolverNewton(), SchemaResolverPhysx()] + if cls._registers_builder_attributes_from_solver(SolverMuJoCo): + resolvers.append(SchemaResolverMjc()) + return resolvers + @classmethod def instantiate_builder_from_stage(cls): """Create builder from USD stage. @@ -1945,7 +1963,7 @@ def instantiate_builder_from_stage(cls): builder = cls.create_builder(up_axis=up_axis) - schema_resolvers = [SchemaResolverNewton(), SchemaResolverPhysx()] + schema_resolvers = cls._get_usd_import_schema_resolvers() # NOTE: None of the add_usd calls below pass joint_ordering or # bodies_follow_joint_ordering, so the live articulation's native diff --git a/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py b/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py index 800ea8b484b1..846585da0dce 100644 --- a/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py +++ b/source/isaaclab_newton/test/cloner/test_newton_builder_world_hook.py @@ -80,7 +80,9 @@ def test_explicit_global_import_uses_global_world(monkeypatch): add_usd = mock.Mock(wraps=builder.add_usd) monkeypatch.setattr(builder, "add_usd", add_usd) manager = SimpleNamespace( - create_builder=mock.Mock(return_value=builder), _inject_terrain_heightfields=mock.Mock(return_value=[]) + create_builder=mock.Mock(return_value=builder), + _get_usd_import_schema_resolvers=NewtonManager._get_usd_import_schema_resolvers, + _inject_terrain_heightfields=mock.Mock(return_value=[]), ) monkeypatch.setattr( replicate_module.PhysicsManager, diff --git a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py index 7ec052c84a83..a03ee0258860 100644 --- a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py +++ b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py @@ -796,6 +796,135 @@ def test_active_manager_create_builder_registers_mpm_attributes(): assert builder.has_custom_attribute("mpm:young_modulus") +@pytest.mark.parametrize("import_path", ["clone", "standalone"]) +@pytest.mark.parametrize( + ("manager_cls", "solver_cfg", "expected_friction", "expected_damping"), + [ + pytest.param(NewtonMJWarpManager, MJWarpSolverCfg(), 0.11, 0.23, id="mjwarp"), + pytest.param(NewtonFeatherstoneManager, FeatherstoneSolverCfg(), 0.0, 0.0, id="featherstone"), + ], +) +def test_production_imports_scope_mujoco_joint_properties( + monkeypatch, import_path, manager_cls, solver_cfg, expected_friction, expected_damping +): + """Only MJWarp imports MuJoCo joint properties through either production path.""" + from isaaclab_newton.cloner.replicate import _build_newton_builder_from_mapping + + from pxr import Sdf, Usd, UsdGeom, UsdPhysics + + stage = Usd.Stage.CreateInMemory() + UsdGeom.SetStageUpAxis(stage, UsdGeom.Tokens.z) + UsdGeom.SetStageMetersPerUnit(stage, 1.0) + physics_prim_path = "/physicsScene" + UsdPhysics.Scene.Define(stage, physics_prim_path) + + root_path = "/Sources/robot" if import_path == "clone" else "/World/robot" + root = UsdGeom.Cube.Define(stage, root_path).GetPrim() + UsdPhysics.RigidBodyAPI.Apply(root) + UsdPhysics.ArticulationRootAPI.Apply(root) + + child_path = f"{root_path}/child" + child = UsdGeom.Cube.Define(stage, child_path).GetPrim() + UsdPhysics.RigidBodyAPI.Apply(child) + + joint = UsdPhysics.RevoluteJoint.Define(stage, f"{child_path}/joint") + joint.CreateAxisAttr().Set("Z") + joint.CreateBody0Rel().SetTargets([root_path]) + joint.CreateBody1Rel().SetTargets([child_path]) + joint.GetPrim().CreateAttribute("mjc:frictionloss", Sdf.ValueTypeNames.Double, True).Set(0.11) + joint.GetPrim().CreateAttribute("mjc:damping", Sdf.ValueTypeNames.Double, True).Set(0.23) + + monkeypatch.setattr( + PhysicsManager, + "_sim", + SimpleNamespace(physics_manager=manager_cls, cfg=SimpleNamespace(physics_prim_path=physics_prim_path)), + ) + monkeypatch.setattr(PhysicsManager, "_cfg", NewtonCfg(solver_cfg=solver_cfg)) + monkeypatch.setattr(PhysicsManager, "_device", "cpu") + monkeypatch.setattr(NewtonManager, "_builder", None) + monkeypatch.setattr(NewtonManager, "_deformable_registry", []) + monkeypatch.setattr(NewtonManager, "_cl_pending_sites", {}) + monkeypatch.setattr(NewtonManager, "_per_world_builder_hooks", []) + monkeypatch.setattr(NewtonManager, "_world_xforms", None) + + if import_path == "clone": + builder, *_ = _build_newton_builder_from_mapping( + stage=stage, + sources=(root_path,), + destinations=("/World/envs/env_{}/robot",), + env_ids=np.array([0], dtype=np.int64), + mapping=np.ones((1, 1), dtype=np.bool_), + load_visual_shapes=False, + ) + else: + monkeypatch.setattr(newton_manager_module, "get_current_stage", lambda: stage) + monkeypatch.setattr( + newton_manager_module, "_restore_visible_colliders_without_visual_shapes", lambda *args: None + ) + monkeypatch.setattr(newton_manager_module, "replace_newton_builder_shape_colors", lambda *args: None) + monkeypatch.setattr(newton_manager_module, "import_builder_visual_material_paths", lambda *args: None) + manager_cls.instantiate_builder_from_stage() + builder = NewtonManager._builder + + model = builder.finalize(device="cpu") + + assert model.joint_friction.numpy()[-1] == pytest.approx(expected_friction) + assert model.joint_damping.numpy()[-1] == pytest.approx(expected_damping) + + +@pytest.mark.parametrize( + ("manager_cls", "imports_mujoco"), + [ + pytest.param(NewtonMJWarpManager, True, id="mjwarp"), + pytest.param(NewtonFeatherstoneManager, False, id="featherstone"), + ], +) +@pytest.mark.parametrize( + "author_newton_values", + [ + pytest.param(False, id="physx-over-mjc"), + pytest.param(True, id="newton-over-physx-over-mjc"), + ], +) +def test_schema_resolver_policy_and_precedence(manager_cls, imports_mujoco, author_newton_values): + """Resolver precedence and MuJoCo fallback selection follow active solver needs.""" + from pxr import Sdf, Usd, UsdGeom, UsdPhysics + + stage = Usd.Stage.CreateInMemory() + root_path = "/World/robot" + root = UsdGeom.Cube.Define(stage, root_path).GetPrim() + UsdPhysics.RigidBodyAPI.Apply(root) + UsdPhysics.ArticulationRootAPI.Apply(root) + child_path = f"{root_path}/child" + child = UsdGeom.Cube.Define(stage, child_path).GetPrim() + UsdPhysics.RigidBodyAPI.Apply(child) + joint = UsdPhysics.RevoluteJoint.Define(stage, f"{child_path}/joint") + joint.CreateAxisAttr().Set("Z") + joint.CreateBody0Rel().SetTargets([root_path]) + joint.CreateBody1Rel().SetTargets([child_path]) + joint_prim = joint.GetPrim() + joint_prim.CreateAttribute("mjc:frictionloss", Sdf.ValueTypeNames.Double, True).Set(0.11) + joint_prim.CreateAttribute("mjc:damping", Sdf.ValueTypeNames.Double, True).Set(0.23) + joint_prim.CreateAttribute("mjc:armature", Sdf.ValueTypeNames.Double, True).Set(0.12) + joint_prim.CreateAttribute("physxJoint:armature", Sdf.ValueTypeNames.Float, True).Set(0.21) + if author_newton_values: + joint_prim.CreateAttribute("newton:friction", Sdf.ValueTypeNames.Double, True).Set(0.31) + joint_prim.CreateAttribute("newton:armature", Sdf.ValueTypeNames.Double, True).Set(0.41) + + schema_resolvers = manager_cls._get_usd_import_schema_resolvers() + builder = ModelBuilder() + manager_cls._register_builder_attributes(builder) + builder.add_usd(stage, schema_resolvers=schema_resolvers) + model = builder.finalize(device="cpu") + + expected_friction = 0.31 if author_newton_values else (0.11 if imports_mujoco else 0.0) + expected_damping = 0.23 if imports_mujoco else 0.0 + expected_armature = 0.41 if author_newton_values else 0.21 + assert model.joint_friction.numpy()[-1] == pytest.approx(expected_friction) + assert model.joint_damping.numpy()[-1] == pytest.approx(expected_damping) + assert model.joint_armature.numpy()[-1] == pytest.approx(expected_armature) + + def test_mpm_end_to_end_with_particle_custom_attributes(): """End-to-end MPM step using ``add_particles(custom_attributes=...)`` — the production path.""" sim_cfg = SimulationCfg( From 5c78cae6dc9b8526c319a40cf5bd825dda195b59 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 7 Sep 2026 15:35:03 -0400 Subject: [PATCH 009/128] Document the 15625 scene partition cap for Isaac RTX renderer (#7573) ## Summary - Document that Isaac RTX's `rtx.scenedb.plugin` caps the number of scene partitions at 15625; requesting more environments than that with scene partitioning enabled silently discards additional partitions and causes environments to share tiled camera views. - Add a warning note to the scene partitioning section of the renderers overview doc. - Add a corresponding Known Issues entry under the Renderers section, following the existing animated-curve scene-partition entry's format. ## Type of change - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Test plan - [x] `uv run isaaclab -f` passes (docs/rst formatting checks) - [x] Manually reviewed rendered structure/headings match existing doc conventions in both files --- .../overview/core-concepts/renderers.rst | 10 ++++++++ docs/source/refs/issues.rst | 24 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/source/overview/core-concepts/renderers.rst b/docs/source/overview/core-concepts/renderers.rst index 942b206fba93..17979729cf11 100644 --- a/docs/source/overview/core-concepts/renderers.rst +++ b/docs/source/overview/core-concepts/renderers.rst @@ -88,6 +88,16 @@ token per instance; markers without that ownership information remain shared. culled once they deform beyond their initial extent. See :ref:`known-issues-animated-curve-scene-partition` for the workaround. +.. warning:: + + The Isaac RTX and OVRTX renderers cap the number of scene partitions at 15625. + Requesting more than 15625 environments with scene partitioning enabled discards + the additional partitions, and ``rtx.scenedb.plugin`` logs ``SceneDbContext : Maximum + number of scene partitions (15625) reached. Additional scene partitions will be + discarded.`` Environments beyond that count then share a partition with another + environment, so their tiled camera views can show another environment's geometry. + See :ref:`known-issues-scene-partition-count-cap` for details. + Architecture Overview --------------------- diff --git a/docs/source/refs/issues.rst b/docs/source/refs/issues.rst index 4e2afe313879..a05adbd2a21b 100644 --- a/docs/source/refs/issues.rst +++ b/docs/source/refs/issues.rst @@ -196,6 +196,30 @@ There are two workarounds: ``False`` to opt out of partitioning entirely, at the cost of the per-environment culling. +.. _known-issues-scene-partition-count-cap: + +Scene partitioning is capped at 15625 partitions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Affects:** ``renderer=isaacsim_rtx`` with scene partitioning enabled, and ``renderer=ovrtx``. + +The underlying ``rtx.scenedb.plugin`` allocates a fixed-size pool of scene partitions and +caps it at 15625, regardless of which renderer requests them. Isaac Lab assigns one scene +partition per environment when +:attr:`~isaaclab_physx.renderers.IsaacRtxRendererCfg.enable_scene_partitioning` is enabled +for the Isaac RTX renderer, and OVRTX always assigns one scene partition per environment, so +runs with more than 15625 environments exceed the pool on either backend. Once the cap is +hit, ``rtx.scenedb.plugin`` logs a warning and discards the remaining partitions: + +.. code-block:: text + + [Warning] [rtx.scenedb.plugin] SceneDbContext : Maximum number of scene partitions + (15625) reached. Additional scene partitions will be discarded. + +Environments beyond the cap are left without their own partition and end up sharing one +with another environment, so their tiled camera views can render another environment's +geometry instead of their own. + Using instanceable assets for markers ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 1ae71258d7e958180996a3bac19657beafc0c1b5 Mon Sep 17 00:00:00 2001 From: Mustafa H <34825877+StafaH@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:31:40 -0400 Subject: [PATCH 010/128] [Tasks] Fix surface gripper observation shapes (#7583) # Description Fix surface-gripper stack observations so `object_grasped` and `object_stacked` return one boolean per environment. The surface-gripper state was reshaped to `(num_envs, 1)` before being combined with an `(num_envs,)` predicate, which caused PyTorch to broadcast the result to `(num_envs, num_envs)` while stepping the environment. No new dependencies are required. Reported in TC_146506 for `IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel` with the Newton Kamino backend. ## Type of change - Bug fix (non-breaking change which fixes an issue) ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable; this corrects an observation tensor shape. ## Validation - `uv run python tools/changelog/cli.py check --include-worktree develop` - `uv run isaaclab -f` ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the pre-commit checks with `uv run isaaclab -f` - [x] Documentation changes are not required for this internal shape correction - [x] My changes generate no new warnings - [x] I have added a changelog fragment under `source/isaaclab_tasks/changelog.d/` - [x] My name already exists in `CONTRIBUTORS.md` --------- Co-authored-by: Kelly Guo --- .../changelog.d/surface-gripper-observation-shape.rst | 5 +++++ .../isaaclab_tasks/contrib/place/mdp/observations.py | 2 +- .../isaaclab_tasks/contrib/stack/mdp/observations.py | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 source/isaaclab_tasks/changelog.d/surface-gripper-observation-shape.rst diff --git a/source/isaaclab_tasks/changelog.d/surface-gripper-observation-shape.rst b/source/isaaclab_tasks/changelog.d/surface-gripper-observation-shape.rst new file mode 100644 index 000000000000..b394ee2c4c1c --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/surface-gripper-observation-shape.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Fixed surface-gripper stack and place observations returning a quadratic environment batch due to unintended + broadcasting. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/place/mdp/observations.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/place/mdp/observations.py index 9a157319667f..75a6fd950b5b 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/place/mdp/observations.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/place/mdp/observations.py @@ -99,7 +99,7 @@ def object_grasped( if hasattr(env.scene, "surface_grippers") and len(env.scene.surface_grippers) > 0: surface_gripper = env.scene.surface_grippers["surface_gripper"] - suction_cup_status = wp.to_torch(surface_gripper.state).view(-1, 1) # 1: closed, 0: closing, -1: open + suction_cup_status = wp.to_torch(surface_gripper.state).view(-1) # 1: closed, 0: closing, -1: open suction_cup_is_closed = (suction_cup_status == 1).to(torch.float32) grasped = torch.logical_and(suction_cup_is_closed, pose_diff < diff_threshold) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/mdp/observations.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/mdp/observations.py index 94adf1b6453f..db480ebfbc04 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/mdp/observations.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/mdp/observations.py @@ -324,7 +324,7 @@ def object_grasped( if hasattr(env.scene, "surface_grippers") and len(env.scene.surface_grippers) > 0: surface_gripper = env.scene.surface_grippers["surface_gripper"] - suction_cup_status = wp.to_torch(surface_gripper.state).view(-1, 1) # 1: closed, 0: closing, -1: open + suction_cup_status = wp.to_torch(surface_gripper.state).view(-1) # 1: closed, 0: closing, -1: open suction_cup_is_closed = (suction_cup_status == 1).to(torch.float32) grasped = torch.logical_and(suction_cup_is_closed, pose_diff < diff_threshold) @@ -369,7 +369,7 @@ def object_stacked( if hasattr(env.scene, "surface_grippers") and len(env.scene.surface_grippers) > 0: surface_gripper = env.scene.surface_grippers["surface_gripper"] - suction_cup_status = wp.to_torch(surface_gripper.state).view(-1, 1) # 1: closed, 0: closing, -1: open + suction_cup_status = wp.to_torch(surface_gripper.state).view(-1) # 1: closed, 0: closing, -1: open suction_cup_is_open = (suction_cup_status == -1).to(torch.float32) stacked = torch.logical_and(suction_cup_is_open, stacked) From aab8730da7321a3c2b701ee29a0d7514b2cd794f Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Mon, 7 Sep 2026 13:39:54 -0700 Subject: [PATCH 011/128] [Backport develop] Fix Franka Reach relative IK teleoperation (#7619) (#7621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Backports #7619 from `release/3.0.0` to `develop`. The change restores native keyboard, gamepad, and SpaceMouse teleoperation for both supported Franka Reach relative-IK configurations: - `physics=isaacsim_physx presets=diffik` - `physics=newton_mjwarp presets=newton_ik` Both configurations accept six-dimensional relative pose actions, so their native device configurations disable the unsupported gripper command. The default joint-position and absolute-pose presets remain unchanged. This is an exact patch replay of merged release commit `17e5f3d51620168806647feeaeab85c2688d2bc4`; no conflict resolution or develop-specific adaptation was required. No new dependencies are required. ## Type of change - Bug fix (non-breaking change which fixes an issue) - Documentation update ## Release backport - [ ] This change originated on `release/3.0.0`; do not backport it again. ## Screenshots Not applicable. ## Validation - Stable patch ID matches the merged #7619 release commit. - `uvx --from pre-commit==4.6.2 pre-commit run --all-files` — passed. - `tools/changelog/cli.py check develop` — passed against the current develop base. - `git diff --check upstream/develop..HEAD` — passed. - Source PR validation for this exact patch: 23 Reach preset tests passed, the Newton random-agent smoke test passed, the documentation build passed without warnings, and `uv run isaaclab -f` passed. The targeted test and documentation commands could not be rerun on this macOS host because the current lockfile supports Linux x86_64/aarch64 and Windows AMD64 only. Develop CI provides the supported-platform verification. ## Checklist - [x] I have read and understood the contribution guidelines. - [x] I have run the available pre-commit checks. - [x] I have included the corresponding documentation change. - [x] My changes generate no new warnings in the available checks. - [x] The source PR's tests cover this exact patch. - [x] I have included the `isaaclab_tasks` changelog fragment. - [x] The original contributor already appears in `CONTRIBUTORS.md`. Co-authored-by: Maximilian Krause <99733341+maxkra15@users.noreply.github.com> --- docs/source/features/isaac_teleop.rst | 3 ++ .../fix-franka-reach-diffik-teleop.rst | 5 +++ .../config/franka/franka_reach_env_cfg.py | 17 ++++++++ .../test/core/test_reach_franka_presets.py | 42 ++++++++++++++++--- 4 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 source/isaaclab_tasks/changelog.d/fix-franka-reach-diffik-teleop.rst diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 70a1c743a67d..780ad0eddbc7 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -861,6 +861,9 @@ follows. * - ``Isaac-Reach-Franka`` with ``physics=isaacsim_physx presets=diffik`` - Keyboard, Gamepad, SpaceMouse - **Arm:** relative IK end-effector control. Gripper disabled. + * - ``Isaac-Reach-Franka`` with ``physics=newton_mjwarp presets=newton_ik`` + - Keyboard, Gamepad, SpaceMouse + - **Arm:** relative Newton IK end-effector control. Gripper disabled. .. note:: diff --git a/source/isaaclab_tasks/changelog.d/fix-franka-reach-diffik-teleop.rst b/source/isaaclab_tasks/changelog.d/fix-franka-reach-diffik-teleop.rst new file mode 100644 index 000000000000..3a61cf601afc --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/fix-franka-reach-diffik-teleop.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Fixed native keyboard, gamepad, and SpaceMouse teleoperation for ``Isaac-Reach-Franka`` with the + ``diffik`` and ``newton_ik`` presets by disabling the unsupported gripper command. diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/reach/config/franka/franka_reach_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/core/reach/config/franka/franka_reach_env_cfg.py index 4a3ceac454ef..27f28ea56160 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/reach/config/franka/franka_reach_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/reach/config/franka/franka_reach_env_cfg.py @@ -14,6 +14,10 @@ import isaaclab.envs.mdp as mdp from isaaclab.controllers.differential_ik_cfg import DifferentialIKControllerCfg +from isaaclab.devices import DevicesCfg +from isaaclab.devices.gamepad import Se3GamepadCfg +from isaaclab.devices.keyboard import Se3KeyboardCfg +from isaaclab.devices.spacemouse import Se3SpaceMouseCfg from isaaclab.envs.mdp.actions.actions_cfg import DifferentialInverseKinematicsActionCfg from isaaclab.utils.configclass import configclass @@ -108,6 +112,19 @@ def __post_init__(self) -> None: # override actions self.actions.arm_action = FrankaArmActionCfg() + # Native SE(3) devices match the 6D relative differential and Newton IK actions. + relative_ik_teleop_devices = DevicesCfg( + devices={ + "keyboard": Se3KeyboardCfg(gripper_term=False, sim_device=self.sim.device), + "gamepad": Se3GamepadCfg(gripper_term=False, sim_device=self.sim.device), + "spacemouse": Se3SpaceMouseCfg(gripper_term=False, sim_device=self.sim.device), + } + ) + self.teleop_devices = preset( + default=DevicesCfg(), + diffik=relative_ik_teleop_devices, + newton_ik=relative_ik_teleop_devices, + ) # override command generator body # end-effector is along z-direction self.commands.ee_pose.body_name = "panda_hand" diff --git a/source/isaaclab_tasks/test/core/test_reach_franka_presets.py b/source/isaaclab_tasks/test/core/test_reach_franka_presets.py index b84a947bf124..b94d4a7b6eea 100644 --- a/source/isaaclab_tasks/test/core/test_reach_franka_presets.py +++ b/source/isaaclab_tasks/test/core/test_reach_franka_presets.py @@ -8,6 +8,7 @@ import pytest import torch from gymnasium.envs.registration import registry +from isaaclab_newton.ik.newton_ik_objectives_cfg import NewtonIKPoseObjectiveCfg import isaaclab.envs.mdp as mdp @@ -28,9 +29,10 @@ def _load_reach_env_cfg(task: str, *presets: str): return resolve_presets(cfg, selected=presets) -def _without_actions(cfg): +def _without_controller_dependent_cfg(cfg): cfg_dict = cfg.to_dict() cfg_dict.pop("actions") + cfg_dict.pop("teleop_devices") return cfg_dict @@ -88,7 +90,7 @@ def test_reach_ur10_physics_presets_change_only_physics(): assert physx_cfg == newton_cfg -def test_reach_action_presets_change_only_the_action_configuration(): +def test_reach_action_presets_preserve_controller_independent_configuration(): joint_pos_physx = _load_env_cfg("joint_pos", "isaacsim_physx") diffik_physx = _load_env_cfg("diffik", "isaacsim_physx") joint_pos_newton = _load_env_cfg("joint_pos", "newton_mjwarp") @@ -96,9 +98,39 @@ def test_reach_action_presets_change_only_the_action_configuration(): newton_ik = _load_env_cfg("newton_ik", "newton_mjwarp") assert _load_env_cfg().actions.arm_action.to_dict() == joint_pos_newton.actions.arm_action.to_dict() - assert _without_actions(joint_pos_physx) == _without_actions(diffik_physx) - assert _without_actions(joint_pos_newton) == _without_actions(diffik_newton) - assert _without_actions(joint_pos_newton) == _without_actions(newton_ik) + assert _without_controller_dependent_cfg(joint_pos_physx) == _without_controller_dependent_cfg(diffik_physx) + assert _without_controller_dependent_cfg(joint_pos_newton) == _without_controller_dependent_cfg(diffik_newton) + assert _without_controller_dependent_cfg(joint_pos_newton) == _without_controller_dependent_cfg(newton_ik) + + +@pytest.mark.parametrize( + ("action_preset", "physics_preset"), + [("diffik", "isaacsim_physx"), ("newton_ik", "newton_mjwarp")], +) +def test_reach_relative_ik_presets_configure_six_dof_native_teleop_devices(action_preset, physics_preset): + cfg = _load_env_cfg(action_preset, physics_preset) + + cfg.validate() + assert set(cfg.teleop_devices.devices) == {"keyboard", "gamepad", "spacemouse"} + assert all(not device_cfg.gripper_term for device_cfg in cfg.teleop_devices.devices.values()) + + +def test_reach_newton_ik_uses_native_se3_command_convention(): + cfg = _load_env_cfg("newton_ik", "newton_mjwarp") + pose_objectives = [ + objective for objective in cfg.actions.arm_action.objectives if isinstance(objective, NewtonIKPoseObjectiveCfg) + ] + + assert len(pose_objectives) == 1 + assert pose_objectives[0].command_type == "pose" + assert pose_objectives[0].use_relative_mode + assert len(pose_objectives[0].scale) == 6 + + +def test_reach_default_preset_does_not_configure_se3_teleop_devices(): + cfg = _load_env_cfg() + + assert cfg.teleop_devices.devices == {} def test_reach_success_requires_position_and_orientation(): From 18086270eecac181e4aad1ae323bde3b9c863072 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Mon, 7 Sep 2026 14:07:43 -0700 Subject: [PATCH 012/128] Correct contrib Newton support in environment browser (#7620) # Description Updates the registry-backed environment browser so it advertises `physics=newton_mjwarp` only for validated contributed task combinations. The following Galbot visuomotor registrations are documented as PhysX-only after nvbug 6695019 reported their shared Newton sensor-initialization failure: - `IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor` - `IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-Joint-Position` - `IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-RmpFlow` The support matrix also documents these tasks as PhysX-only: - `IsaacContrib-Factory-Franka` - `IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel` The UR10 task uses the PhysX-only surface-gripper implementation and already rejects Newton during configuration validation. Renderer selectors remain available for the Galbot tasks because renderer selection is independent of the physics backend. Regenerating the browser also captures existing registry drift for the two DR Legs tasks, whose PhysX and Newton Kamino presets were missing from the generated rows. Focused regression coverage verifies every newly excluded task while preserving supported task combinations. ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) - [x] Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable; this updates generated environment metadata and command options. ## Validation - `uv run --frozen python -m pytest --confcutdir=tools/test tools/test/test_environ_docs.py -q` (33 passed) - `uv run --frozen python tools/update_environments_rst.py --check` - `uv run --isolated --extra dev --extra ov -- make -C docs current-docs SPHINXOPTS=-q` - `SKIP=check-changelog-fragments uv run --frozen isaaclab -f` (all applicable hooks passed; no source package changed) - `git diff --check` ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the applicable pre-commit checks - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective - [x] No source package changed, so no changelog fragment is required - [ ] I have added my name to `CONTRIBUTORS.md` or my name already exists there --- .../source/_static/css/environment-browser.js | 14 +++++----- tools/environ_docs.py | 5 ++++ tools/test/test_environ_docs.py | 27 ++++++++++++++++--- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index fd49063fc3df..1d7f5168a0bf 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -72,10 +72,10 @@ ["IsaacContrib-Deploy-Reach-Rizon4s-ROS-Inference", "rsl_rl", "", "", ""], ["IsaacContrib-Deploy-Reach-UR10e", "rsl_rl", "", "", "", {}, "tasks/manipulation/ur10e_reach.jpg"], ["IsaacContrib-Deploy-Reach-UR10e-ROS-Inference", "rsl_rl", "", "", "", {}, "tasks/manipulation/ur10e_reach.jpg"], - ["IsaacContrib-DrLegs-HoldPose", "rsl_rl", "", "", "", {}, "tasks/locomotion/dr_legs.jpg"], - ["IsaacContrib-DrLegs-Walk", "rsl_rl", "", "", "", {}, "tasks/locomotion/dr_legs.jpg"], + ["IsaacContrib-DrLegs-HoldPose", "rsl_rl", "isaacsim_physx,newton_kamino", "", "", {}, "tasks/locomotion/dr_legs.jpg"], + ["IsaacContrib-DrLegs-Walk", "rsl_rl", "isaacsim_physx,newton_kamino", "", "", {}, "tasks/locomotion/dr_legs.jpg"], ["IsaacContrib-ExhaustPipe-GR1T2-Pink-IK-Abs", "", "", "", ""], - ["IsaacContrib-Factory-Franka", "rsl_rl", "isaacsim_physx,newton_mjwarp", "", "accumulator,choice,gear_mesh_large,gear_mesh_medium,gear_mesh_small,nut_thread_m16,peg_insert_12mm,peg_insert_16mm,peg_insert_4mm,peg_insert_8mm,rod_insert_12mm,rod_insert_16mm,rod_insert_4mm,rod_insert_8mm"], + ["IsaacContrib-Factory-Franka", "rsl_rl", "isaacsim_physx", "", "accumulator,choice,gear_mesh_large,gear_mesh_medium,gear_mesh_small,nut_thread_m16,peg_insert_12mm,peg_insert_16mm,peg_insert_4mm,peg_insert_8mm,rod_insert_12mm,rod_insert_16mm,rod_insert_4mm,rod_insert_8mm"], ["IsaacContrib-Factory-GearMesh-Direct", "rl_games", "", "", "", {}, "tasks/factory/gear_mesh.jpg"], ["IsaacContrib-Factory-NutThread-Direct", "rl_games", "", "", "", {}, "tasks/factory/nut_thread.jpg"], ["IsaacContrib-Factory-PegInsert-Direct", "rl_games", "", "", "", {}, "tasks/factory/peg_insert.jpg"], @@ -120,9 +120,9 @@ ["IsaacContrib-Stack-Cube-Franka-IK-Rel-Visuomotor", "", "isaacsim_physx,newton_mjwarp", "", "", {}, "tasks/manipulation/franka_stack.jpg"], ["IsaacContrib-Stack-Cube-Franka-IK-Rel-Visuomotor-Cosmos", "", "isaacsim_physx,newton_mjwarp", "", "", {}, "tasks/manipulation/franka_stack.jpg"], ["IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-RmpFlow", "", "isaacsim_physx", "", "", {}, "tasks/manipulation/galbot_stack_cube.jpg"], - ["IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor", "", "isaacsim_physx,newton_mjwarp", "isaacsim_rtx,newton_renderer,ovrtx", ""], - ["IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-Joint-Position", "", "isaacsim_physx,newton_mjwarp", "isaacsim_rtx,newton_renderer,ovrtx", ""], - ["IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-RmpFlow", "", "isaacsim_physx,newton_mjwarp", "isaacsim_rtx,newton_renderer,ovrtx", ""], + ["IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor", "", "isaacsim_physx", "isaacsim_rtx,newton_renderer,ovrtx", ""], + ["IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-Joint-Position", "", "isaacsim_physx", "isaacsim_rtx,newton_renderer,ovrtx", ""], + ["IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-RmpFlow", "", "isaacsim_physx", "isaacsim_rtx,newton_renderer,ovrtx", ""], ["IsaacContrib-Stack-Cube-Galbot-Right-Arm-Suction-RmpFlow", "", "isaacsim_physx", "", ""], ["IsaacContrib-Stack-Cube-Instance-Randomize-Franka", "", "", "", ""], ["IsaacContrib-Stack-Cube-Instance-Randomize-Franka-IK-Rel", "", "", "", ""], @@ -131,7 +131,7 @@ ["IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", "", "isaacsim_physx", "", ""], ["IsaacContrib-Stack-Cube-SO101-Joint-Teleop-v0", "", "isaacsim_physx", "", ""], ["IsaacContrib-Stack-Cube-SO101-v0", "", "isaacsim_physx", "", ""], - ["IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel", "", "isaacsim_physx,newton_mjwarp", "", "", {}, "tasks/manipulation/ur10_stack_surface_gripper.jpg"], + ["IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel", "", "isaacsim_physx", "", "", {}, "tasks/manipulation/ur10_stack_surface_gripper.jpg"], ["IsaacContrib-Stack-Cube-UR10-Short-Suction-IK-Rel", "", "isaacsim_physx", "", "", {}, "tasks/manipulation/ur10_stack_surface_gripper.jpg"], ["IsaacContrib-TrackPositionNoObstacles-ARL-Robot-1", "rl_games,rsl_rl,skrl", "", "", "", {}, "tasks/drone_arl/arl_robot_1_track_position_state_based.jpg"], ["IsaacContrib-Tracking-LocoManip-Digit", "rsl_rl", "isaacsim_physx", "", "", {}, "tasks/locomotion/agility_digit_loco_manip.jpg"], diff --git a/tools/environ_docs.py b/tools/environ_docs.py index 54e648bdae3e..28d21be751d8 100644 --- a/tools/environ_docs.py +++ b/tools/environ_docs.py @@ -89,6 +89,7 @@ # commands until the corresponding task supports them end to end. _NEWTON_MJWARP_EXCLUSIONS = frozenset( { + "IsaacContrib-Factory-Franka", "IsaacContrib-Place-Mug-Agibot-Left-Arm-RmpFlow", "IsaacContrib-Place-Toy2Box-Agibot-Right-Arm-RmpFlow", "IsaacContrib-Stack-Cube-Bin-Franka-IK-Rel-Mimic", @@ -99,12 +100,16 @@ "IsaacContrib-Stack-Cube-Franka-IK-Rel", "IsaacContrib-Stack-Cube-Franka-IK-Rel-Skillgen", "IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-RmpFlow", + "IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor", + "IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-Joint-Position", + "IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-RmpFlow", "IsaacContrib-Stack-Cube-Galbot-Right-Arm-Suction-RmpFlow", "IsaacContrib-Stack-Cube-RedGreen-Franka-IK-Rel", "IsaacContrib-Stack-Cube-RedGreenBlue-Franka-IK-Rel", "IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", "IsaacContrib-Stack-Cube-SO101-Joint-Teleop-v0", "IsaacContrib-Stack-Cube-SO101-v0", + "IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel", "IsaacContrib-Stack-Cube-UR10-Short-Suction-IK-Rel", } ) diff --git a/tools/test/test_environ_docs.py b/tools/test/test_environ_docs.py index 969c342a6791..1bcb6531e3cd 100644 --- a/tools/test/test_environ_docs.py +++ b/tools/test/test_environ_docs.py @@ -201,21 +201,42 @@ def test_physics_names_for_docs_infers_physx_from_default(): assert names == ["newton_mjwarp", "physx"] -def test_preset_exclusions_remove_only_runtime_disabled_task_combinations(): +@pytest.mark.parametrize( + "task_name", + [ + "IsaacContrib-Factory-Franka", + "IsaacContrib-Stack-Cube-Franka", + "IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor", + "IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-Joint-Position", + "IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-RmpFlow", + "IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel", + ], +) +def test_preset_exclusions_remove_runtime_disabled_task_combinations(task_name: str): presets = { PresetTarget.PHYSICS: ["isaacsim_physx", "newton_mjwarp"], PresetTarget.RENDERER: ["isaacsim_rtx", "newton_renderer"], PresetTarget.DOMAIN: ["rgb"], } - excluded = _apply_preset_exclusions("IsaacContrib-Stack-Cube-Franka", presets) - unchanged = _apply_preset_exclusions("Isaac-Lift-Franka", presets) + excluded = _apply_preset_exclusions(task_name, presets) assert excluded == { PresetTarget.PHYSICS: ["isaacsim_physx"], PresetTarget.RENDERER: ["isaacsim_rtx", "newton_renderer"], PresetTarget.DOMAIN: ["rgb"], } + + +def test_preset_exclusions_keep_supported_task_combinations(): + presets = { + PresetTarget.PHYSICS: ["isaacsim_physx", "newton_mjwarp"], + PresetTarget.RENDERER: ["isaacsim_rtx", "newton_renderer"], + PresetTarget.DOMAIN: ["rgb"], + } + + unchanged = _apply_preset_exclusions("Isaac-Lift-Franka", presets) + assert unchanged == presets From 6513de14aecbe7a856e769a10de63bb93fd12ba1 Mon Sep 17 00:00:00 2001 From: "isaaclab-bot[bot]" <282401363+isaaclab-bot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 15:51:50 -0700 Subject: [PATCH 013/128] [CI] Bump Isaac Sim image to b6222dffc018 (#7600) This automated draft updates CI to the current Isaac Sim nightly image. | Field | Value | |---|---| | Image | `nvcr.io/0947644777160149/internal/isaac-sim` | | Moving tag | `latest-develop` | | Current pin | `latest-develop@sha256:0bd319db2e50e667e75abf897ec48c2315e6707fed4faa6bb514bf393e537f8a` | | Candidate pin | `latest-develop@sha256:b6222dffc0182e82f49d656d08dabdfb6a279e368a478cbd38080bbe959bb2ba` | Source: https://registry.ngc.nvidia.com/orgs/0947644777160149/teams/internal/containers/isaac-sim/tags New PRs are opened as drafts so maintainers can merge after the CI results are acceptable. ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` --------- Co-authored-by: isaaclab-bot[bot] <282401363+isaaclab-bot[bot]@users.noreply.github.com> Co-authored-by: Kelly Guo --- .github/workflows/config.yaml | 2 +- .../changelog.d/spacemouse-partial-init-shutdown.rst | 5 +++++ .../isaaclab/devices/spacemouse/se3_spacemouse.py | 4 +++- .../test/devices/test_device_constructors.py | 7 +++++++ .../changelog.d/kitless-pxr-thread-limit.skip | 0 source/isaaclab_ov/test/conftest.py | 12 ++++++++++++ 6 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 source/isaaclab/changelog.d/spacemouse-partial-init-shutdown.rst create mode 100644 source/isaaclab_ov/changelog.d/kitless-pxr-thread-limit.skip create mode 100644 source/isaaclab_ov/test/conftest.py diff --git a/.github/workflows/config.yaml b/.github/workflows/config.yaml index 353d4a65be4f..769bada880a4 100644 --- a/.github/workflows/config.yaml +++ b/.github/workflows/config.yaml @@ -10,6 +10,6 @@ # which the CI credential can reach. isaacsim_image_name: nvcr.io/0947644777160149/internal/isaac-sim # Isaac Sim 6.1.0-alpha.50 (b86cf6ce) includes Kit 110.3.0-360924's fix for NVBug 6566677. -isaacsim_image_tag: latest-develop@sha256:0bd319db2e50e667e75abf897ec48c2315e6707fed4faa6bb514bf393e537f8a +isaacsim_image_tag: latest-develop@sha256:b6222dffc0182e82f49d656d08dabdfb6a279e368a478cbd38080bbe959bb2ba isaaclab_image_name: nvcr.io/0947644777160149/internal/isaac-lab ovphysx_wheelhouse_image: "" diff --git a/source/isaaclab/changelog.d/spacemouse-partial-init-shutdown.rst b/source/isaaclab/changelog.d/spacemouse-partial-init-shutdown.rst new file mode 100644 index 000000000000..8104eb2d74cb --- /dev/null +++ b/source/isaaclab/changelog.d/spacemouse-partial-init-shutdown.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Fixed ``Se3SpaceMouse`` cleanup raising an exception when device initialization failed before + starting its listener thread. diff --git a/source/isaaclab/isaaclab/devices/spacemouse/se3_spacemouse.py b/source/isaaclab/isaaclab/devices/spacemouse/se3_spacemouse.py index 243f8eb6483f..49ce997da486 100644 --- a/source/isaaclab/isaaclab/devices/spacemouse/se3_spacemouse.py +++ b/source/isaaclab/isaaclab/devices/spacemouse/se3_spacemouse.py @@ -85,7 +85,9 @@ def __init__(self, cfg: Se3SpaceMouseCfg): def __del__(self): """Destructor for the class.""" - self._thread.join() + thread = getattr(self, "_thread", None) + if thread is not None and thread.is_alive(): + thread.join() def __str__(self) -> str: """Returns: A string containing the information of joystick.""" diff --git a/source/isaaclab/test/devices/test_device_constructors.py b/source/isaaclab/test/devices/test_device_constructors.py index aa754ea7fd74..25e5ecc62e4d 100644 --- a/source/isaaclab/test/devices/test_device_constructors.py +++ b/source/isaaclab/test/devices/test_device_constructors.py @@ -356,6 +356,13 @@ def test_se3spacemouse_constructors(mock_environment, mocker): assert result.shape == (7,) # (pos_x, pos_y, pos_z, rot_x, rot_y, rot_z, gripper) +def test_se3spacemouse_destructor_handles_partial_initialization(): + """The destructor must tolerate construction failing before the listener thread exists.""" + spacemouse = Se3SpaceMouse.__new__(Se3SpaceMouse) + + spacemouse.__del__() + + """ Test Haply devices. """ diff --git a/source/isaaclab_ov/changelog.d/kitless-pxr-thread-limit.skip b/source/isaaclab_ov/changelog.d/kitless-pxr-thread-limit.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ov/test/conftest.py b/source/isaaclab_ov/test/conftest.py new file mode 100644 index 000000000000..5f460add94f7 --- /dev/null +++ b/source/isaaclab_ov/test/conftest.py @@ -0,0 +1,12 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared configuration for kitless OVPhysX tests.""" + +import os + +# TODO: Remove once usd-core>=26.5 is the minimum. Earlier releases can corrupt +# the heap when OpenUSD parses payloads concurrently in kitless processes. +os.environ["PXR_WORK_THREAD_LIMIT"] = "1" From 63d579e93388bf6c6f59c6866c36c0e2c12c0b0d Mon Sep 17 00:00:00 2001 From: Mustafa H <34825877+StafaH@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:53:03 -0400 Subject: [PATCH 014/128] [Tests] Skip DrLegs-Walk in the contrib environment test (#7623) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description `test-contrib-environments` failed on #7610 (run [34042243157](https://github.com/isaac-sim/IsaacLab/actions/runs/34042243157/job/101511058208)) in `test_contrib_environments[IsaacContrib-DrLegs-Walk]` with `AssertionError: Invalid data` on the policy observation. The PR itself only touched `source/isaaclab/test/install_ci`, so this is unrelated to that change. **Root cause.** The first row of the failing observation is NaN across projected gravity, base angular velocity and joint positions, i.e. the robot state itself is non-finite, not a single observation term. Reproducing the test scenario locally (2 envs, 20 random-action steps, fresh env per seed) shows the Kamino P-ADMM solver diverging mid-episode: in 1 of 12 seeds the entire state of env 0 (root position, root quaternion, joint positions and velocities) turns NaN at step 6. The heading-driven velocity command then inherits the NaN through `heading_w`, and the `root_height` / `bad_orientation` termination terms cannot fire because every comparison against NaN is false, so the episode is never reset. The sibling `IsaacContrib-DrLegs-HoldPose` task (same robot, same solver preset, same random actions) stayed finite in 9 of 9 local trials and passed in the same CI job, as did the deprecated alias id of the Walk task, which is why the failure is intermittent rather than deterministic. **Change.** Add `DrLegs-Walk` to `_SKIPPED_TASK_SUBSTRINGS` in `test_contrib_environments.py` with the reason documented inline. The substring covers both `IsaacContrib-DrLegs-Walk` and its deprecated alias `Isaac-DrLegs-Walk-v0`, which the test also enumerates. HoldPose keeps running. This is a test-side skip; the solver instability itself needs to be addressed in Kamino/Newton (or by retuning the DR Legs solver preset), and the inline comment says to re-enable once that lands. ## Validation - Reproduction script (launches once, rebuilds the env per seed, mirrors `env_test_utils._run_environments`): Walk 1/12 seeds NaN at step 6 with root pose, joint state and the velocity command all non-finite; HoldPose 0/9. - `pytest source/isaaclab_tasks/test/contrib/test_contrib_environments.py -k DrLegs-Walk -rs`: both ids report `SKIPPED ... Kamino solver intermittently produces NaN robot state under random actions.`; HoldPose ids still collect as runnable. - Ruff and ruff-format pass; `isaaclab_tasks` changelog `.skip` fragment included. ## Type of change - Test fix (non-breaking) ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package (do **not** edit `CHANGELOG.rst` or bump `extension.toml` — CI handles that) - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- .../changelog.d/fix-skip-drlegs-walk-contrib-test.skip | 3 +++ .../isaaclab_tasks/test/contrib/test_contrib_environments.py | 5 +++++ 2 files changed, 8 insertions(+) create mode 100644 source/isaaclab_tasks/changelog.d/fix-skip-drlegs-walk-contrib-test.skip diff --git a/source/isaaclab_tasks/changelog.d/fix-skip-drlegs-walk-contrib-test.skip b/source/isaaclab_tasks/changelog.d/fix-skip-drlegs-walk-contrib-test.skip new file mode 100644 index 000000000000..e29cd024514a --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/fix-skip-drlegs-walk-contrib-test.skip @@ -0,0 +1,3 @@ +Test-only change: skip ``IsaacContrib-DrLegs-Walk`` (and its deprecated alias) in the contributed +environment smoke test because the Kamino solver intermittently diverges to NaN under random actions. +No user-visible behavior change. diff --git a/source/isaaclab_tasks/test/contrib/test_contrib_environments.py b/source/isaaclab_tasks/test/contrib/test_contrib_environments.py index 25437c653e3c..3d2b961e244d 100644 --- a/source/isaaclab_tasks/test/contrib/test_contrib_environments.py +++ b/source/isaaclab_tasks/test/contrib/test_contrib_environments.py @@ -32,6 +32,11 @@ "IsaacContrib-AutoMate-Disassembly-Direct": "Requires CUDA support outside the standard environment test runner.", } _SKIPPED_TASK_SUBSTRINGS = { + # Under random actions the Kamino P-ADMM solver intermittently diverges and the whole robot state + # (root pose, joint state) turns NaN mid-episode, so the run fails nondeterministically (about 1 in 12 + # seeds locally; the sibling HoldPose task stays finite). The termination terms cannot catch a NaN state. + # Re-enable once the solver instability is resolved upstream. + "DrLegs-Walk": "Kamino solver intermittently produces NaN robot state under random actions.", "RmpFlow": "Uses SingleArticulation, which requires an update.", "Skillgen": "Requires cuRobo-specific coverage.", "Suction": "Requires CPU simulation.", From 6fd0f1ef81a3035e6fd86f90dda55e1cfd2a45f3 Mon Sep 17 00:00:00 2001 From: Mustafa H <34825877+StafaH@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:03:35 -0400 Subject: [PATCH 015/128] [Tasks] Fix pretrained checkpoint lookup for domain presets (#7594) # Description Published checkpoint lookup included the task, physics backend, renderer backend, and RL library, but ignored domain presets. As a result, a depth policy could not be found and an unsuffixed RGB policy could be selected for an incompatible preset. This change: - adds non-default domain presets to the canonical checkpoint filename; - keeps aliases of the default preset, such as RGB, on the existing unsuffixed filename; - derives the selection centrally from the existing preset overrides, covering all checkpoint consumers; - declares the published Cartpole depth checkpoint and disables the environment-browser pretrained option for undeclared preset selections. No new dependencies. ## Type of change - Bug fix (non-breaking change which fixes an issue) - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into develop ## Screenshots Not applicable; the UI change disables the existing pretrained-checkpoint checkbox for unsupported preset selections. ## Validation - 17 existing/new pretrained-checkpoint tests passed. - 28 existing/new environment documentation tests passed. - Environment browser generation check passed for 137 registered training environments. - Ruff lint and formatting hooks passed on all touched Python files. - The repository-wide format command passed every hook except the changelog-fragment comparison, which reports pre-existing develop/base differences in unrelated isaaclab_newton, isaaclab_experimental, isaaclab_tasks, and isaaclab_visualizers files. ## Checklist - [x] I have read and understood the contribution guidelines - [ ] I have run the complete pre-commit checks without unrelated base failures - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove the fix is effective - [x] I have added a changelog fragment for every touched package - [x] My name already exists in CONTRIBUTORS.md --- .../source/_static/css/environment-browser.js | 52 +++--- .../test_train_and_publish_checkpoints.py | 35 +++- .../tools/train_and_publish_checkpoints.py | 80 ++++++--- .../mustafa-pretrained-checkpoint-presets.rst | 6 + .../utils/pretrained_checkpoint.py | 153 ++++++++++++++++-- .../test/test_pretrained_checkpoint.py | 42 ++++- ...mustafa-pretrained-checkpoint-presets.skip | 1 + .../isaaclab_tasks/core/cartpole/__init__.py | 1 + tools/environ_docs.py | 63 +++++++- tools/test/test_environ_docs.py | 16 +- 10 files changed, 368 insertions(+), 81 deletions(-) create mode 100644 source/isaaclab_rl/changelog.d/mustafa-pretrained-checkpoint-presets.rst create mode 100644 source/isaaclab_tasks/changelog.d/mustafa-pretrained-checkpoint-presets.skip diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index 1d7f5168a0bf..86f9a745e306 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -16,36 +16,36 @@ ["Isaac-Ant", "rl_games,rsl_rl,skrl,sb3", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/ant.jpg", true], ["Isaac-Cartpole-Direct", "rl_games,rsl_rl,skrl,sb3", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/cartpole.jpg", true], ["Isaac-Cartpole", "rl_games,rsl_rl,skrl,sb3", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/cartpole.jpg", true], - ["Isaac-Cartpole-Camera-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl", {}, "tasks/classic/cartpole.jpg"], - ["Isaac-Cartpole-Camera", "rl_games,rsl_rl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,resnet18,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl,theia_tiny", {"rl_games_cfg_entry_point": ["albedo", "depth", "rgb", "semantic_segmentation", "simple_shading_constant_diffuse", "simple_shading_diffuse_mdl", "simple_shading_full_mdl"], "rl_games_feature_cfg_entry_point": ["resnet18", "theia_tiny"], "rsl_rl_cfg_entry_point": ["albedo", "depth", "rgb", "semantic_segmentation", "simple_shading_constant_diffuse", "simple_shading_diffuse_mdl", "simple_shading_full_mdl"], "rsl_rl_feature_cfg_entry_point": ["resnet18", "theia_tiny"]}, "tasks/classic/cartpole.jpg"], + ["Isaac-Cartpole-Camera-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl", {}, "tasks/classic/cartpole.jpg", false, {"*": ["rgb"], "rl_games": ["depth"]}], + ["Isaac-Cartpole-Camera", "rl_games,rsl_rl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,resnet18,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl,theia_tiny", {"rl_games_cfg_entry_point": ["albedo", "depth", "rgb", "semantic_segmentation", "simple_shading_constant_diffuse", "simple_shading_diffuse_mdl", "simple_shading_full_mdl"], "rl_games_feature_cfg_entry_point": ["resnet18", "theia_tiny"], "rsl_rl_cfg_entry_point": ["albedo", "depth", "rgb", "semantic_segmentation", "simple_shading_constant_diffuse", "simple_shading_diffuse_mdl", "simple_shading_full_mdl"], "rsl_rl_feature_cfg_entry_point": ["resnet18", "theia_tiny"]}, "tasks/classic/cartpole.jpg", false, {"*": ["rgb"]}], ["Isaac-Fourbar-Pole-Swingup", "rsl_rl", "newton_kamino", "", "", {}, "tasks/classic/fourbar_pole.jpg"], ["Isaac-Humanoid-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/humanoid.jpg", true], ["Isaac-Humanoid", "rl_games,rsl_rl,skrl,sb3", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/humanoid.jpg", true], - ["Isaac-Lift-Cable-Franka", "rsl_rl", "newton_mjwarp_vbd_proxy", "", "ik,joint", {}, "tasks/manipulation/franka_lift_cable.jpg"], - ["Isaac-Lift-Cable-Franka-Camera", "rsl_rl", "newton_mjwarp_vbd_proxy", "isaacsim_rtx,newton_renderer,ovrtx", "ik,joint", {}, "tasks/manipulation/franka_lift_cable.jpg"], - ["Isaac-Lift-Cloth-Franka", "rsl_rl", "isaacsim_physx,newton_mjwarp_vbd_proxy", "", "ik,joint", {}, "tasks/manipulation/franka_lift_cloth.jpg"], - ["Isaac-Lift-Cloth-Franka-Camera", "rsl_rl", "isaacsim_physx,newton_mjwarp_vbd_proxy", "isaacsim_rtx,newton_renderer,ovrtx", "ik,joint", {}, "tasks/manipulation/franka_lift_cloth.jpg"], - ["Isaac-Lift-Franka", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "cube,shapes"], - ["Isaac-Lift-KukaAllegro", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "cube,shapes", {}, "tasks/manipulation/kuka_allegro_lift.jpg"], - ["Isaac-Lift-KukaAllegro-Camera", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo128,albedo256,albedo64,cube,depth128,depth256,depth64,duo_camera,raycaster_depth128,raycaster_depth256,raycaster_depth64,rgb128,rgb256,rgb64,semantic_segmentation128,semantic_segmentation256,semantic_segmentation64,shapes,simple_shading_constant_diffuse128,simple_shading_constant_diffuse256,simple_shading_constant_diffuse64,simple_shading_diffuse_mdl128,simple_shading_diffuse_mdl256,simple_shading_diffuse_mdl64,simple_shading_full_mdl128,simple_shading_full_mdl256,simple_shading_full_mdl64,single_camera", {}, "tasks/manipulation/kuka_allegro_lift.jpg"], - ["Isaac-Lift-Soft-Franka", "rsl_rl", "isaacsim_physx,newton_mjwarp_vbd_proxy", "", "ik,joint", {}, "newton/franka-mjwarp-vbd-coupling.png"], - ["Isaac-Lift-Soft-Franka-Camera", "rsl_rl", "isaacsim_physx,newton_mjwarp_vbd_proxy", "isaacsim_rtx,newton_renderer,ovrtx", "ik,joint", {}, "newton/franka-mjwarp-vbd-coupling.png"], + ["Isaac-Lift-Cable-Franka", "rsl_rl", "newton_mjwarp_vbd_proxy", "", "ik,joint", {}, "tasks/manipulation/franka_lift_cable.jpg", false, {"*": ["joint"]}], + ["Isaac-Lift-Cable-Franka-Camera", "rsl_rl", "newton_mjwarp_vbd_proxy", "isaacsim_rtx,newton_renderer,ovrtx", "ik,joint", {}, "tasks/manipulation/franka_lift_cable.jpg", false, {"*": ["joint"]}], + ["Isaac-Lift-Cloth-Franka", "rsl_rl", "isaacsim_physx,newton_mjwarp_vbd_proxy", "", "ik,joint", {}, "tasks/manipulation/franka_lift_cloth.jpg", false, {"*": ["joint"]}], + ["Isaac-Lift-Cloth-Franka-Camera", "rsl_rl", "isaacsim_physx,newton_mjwarp_vbd_proxy", "isaacsim_rtx,newton_renderer,ovrtx", "ik,joint", {}, "tasks/manipulation/franka_lift_cloth.jpg", false, {"*": ["joint"]}], + ["Isaac-Lift-Franka", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "cube,shapes", {}, "", false, {"*": ["shapes"]}], + ["Isaac-Lift-KukaAllegro", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "cube,shapes", {}, "tasks/manipulation/kuka_allegro_lift.jpg", false, {"*": ["shapes"]}], + ["Isaac-Lift-KukaAllegro-Camera", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo128,albedo256,albedo64,cube,depth128,depth256,depth64,duo_camera,raycaster_depth128,raycaster_depth256,raycaster_depth64,rgb128,rgb256,rgb64,semantic_segmentation128,semantic_segmentation256,semantic_segmentation64,shapes,simple_shading_constant_diffuse128,simple_shading_constant_diffuse256,simple_shading_constant_diffuse64,simple_shading_diffuse_mdl128,simple_shading_diffuse_mdl256,simple_shading_diffuse_mdl64,simple_shading_full_mdl128,simple_shading_full_mdl256,simple_shading_full_mdl64,single_camera", {}, "tasks/manipulation/kuka_allegro_lift.jpg", false, {"*": ["rgb64", "shapes", "single_camera"]}], + ["Isaac-Lift-Soft-Franka", "rsl_rl", "isaacsim_physx,newton_mjwarp_vbd_proxy", "", "ik,joint", {}, "newton/franka-mjwarp-vbd-coupling.png", false, {"*": ["joint"]}], + ["Isaac-Lift-Soft-Franka-Camera", "rsl_rl", "isaacsim_physx,newton_mjwarp_vbd_proxy", "isaacsim_rtx,newton_renderer,ovrtx", "ik,joint", {}, "newton/franka-mjwarp-vbd-coupling.png", false, {"*": ["joint"]}], ["Isaac-Open-Drawer-Franka-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/franka_open_drawer.jpg"], ["Isaac-Open-Drawer-Franka", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/franka_open_drawer.jpg"], - ["Isaac-Pendulum-MARL-Direct", "rl_games,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/cart_double_pendulum.jpg", false, {"skrl": "MAPPO"}], - ["Isaac-Reach-Franka", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "diffik,diffik_abs,joint_pos,newton_ik", {}, "tasks/manipulation/franka_reach.jpg", true], + ["Isaac-Pendulum-MARL-Direct", "rl_games,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/cart_double_pendulum.jpg", false, {}, {"skrl": "MAPPO"}], + ["Isaac-Reach-Franka", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "diffik,diffik_abs,joint_pos,newton_ik", {}, "tasks/manipulation/franka_reach.jpg", true, {"*": ["joint_pos"]}], ["Isaac-Reach-Franka-OSC", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "diffik_abs", {}, "tasks/manipulation/franka_reach.jpg"], ["Isaac-Reach-UR10", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/ur10_reach.jpg", true], ["Isaac-Reorient-Cube-Allegro-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/allegro_cube.jpg", true], - ["Isaac-Reorient-Cube-Allegro", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "randomized,reset_only", {}, "tasks/manipulation/allegro_cube.jpg"], + ["Isaac-Reorient-Cube-Allegro", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "randomized,reset_only", {}, "tasks/manipulation/allegro_cube.jpg", false, {"*": ["reset_only"]}], ["Isaac-Reorient-Cube-Shadow-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/shadow_cube.jpg"], ["Isaac-Reorient-Cube-Shadow", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "asymmetric,randomized"], - ["Isaac-Reorient-Cube-Shadow-Camera-Direct", "rl_games,rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,full,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl", {}, "tasks/manipulation/shadow_cube.jpg"], - ["Isaac-Reorient-Cube-Shadow-Camera", "rl_games,rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,full,randomized,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl"], - ["Isaac-Reorient-Franka", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "cube,shapes"], - ["Isaac-Reorient-KukaAllegro", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "cube,shapes", {}, "tasks/manipulation/kuka_allegro_reorient.jpg"], - ["Isaac-Reorient-KukaAllegro-Camera", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo128,albedo256,albedo64,cube,depth128,depth256,depth64,duo_camera,raycaster_depth128,raycaster_depth256,raycaster_depth64,rgb128,rgb256,rgb64,semantic_segmentation128,semantic_segmentation256,semantic_segmentation64,shapes,simple_shading_constant_diffuse128,simple_shading_constant_diffuse256,simple_shading_constant_diffuse64,simple_shading_diffuse_mdl128,simple_shading_diffuse_mdl256,simple_shading_diffuse_mdl64,simple_shading_full_mdl128,simple_shading_full_mdl256,simple_shading_full_mdl64,single_camera", {}, "tasks/manipulation/kuka_allegro_reorient.jpg"], - ["Isaac-Shadow-Handover-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/shadow_hand_over.jpg", false, {"skrl": "MAPPO"}], + ["Isaac-Reorient-Cube-Shadow-Camera-Direct", "rl_games,rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,full,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl", {}, "tasks/manipulation/shadow_cube.jpg", false, {"*": ["full"]}], + ["Isaac-Reorient-Cube-Shadow-Camera", "rl_games,rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,full,randomized,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl", {}, "", false, {"*": ["full"]}], + ["Isaac-Reorient-Franka", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "cube,shapes", {}, "", false, {"*": ["shapes"]}], + ["Isaac-Reorient-KukaAllegro", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "cube,shapes", {}, "tasks/manipulation/kuka_allegro_reorient.jpg", false, {"*": ["shapes"]}], + ["Isaac-Reorient-KukaAllegro-Camera", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo128,albedo256,albedo64,cube,depth128,depth256,depth64,duo_camera,raycaster_depth128,raycaster_depth256,raycaster_depth64,rgb128,rgb256,rgb64,semantic_segmentation128,semantic_segmentation256,semantic_segmentation64,shapes,simple_shading_constant_diffuse128,simple_shading_constant_diffuse256,simple_shading_constant_diffuse64,simple_shading_diffuse_mdl128,simple_shading_diffuse_mdl256,simple_shading_diffuse_mdl64,simple_shading_full_mdl128,simple_shading_full_mdl256,simple_shading_full_mdl64,single_camera", {}, "tasks/manipulation/kuka_allegro_reorient.jpg", false, {"*": ["rgb64", "shapes", "single_camera"]}], + ["Isaac-Shadow-Handover-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/manipulation/shadow_hand_over.jpg", false, {}, {"skrl": "MAPPO"}], ["Isaac-Shadow-Handover", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "randomized"], ["Isaac-Velocity-Flat-AnymalD", "rsl_rl,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/locomotion/anymal_d_flat.jpg", true], ["Isaac-Velocity-Flat-Cassie", "rsl_rl,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "", true], @@ -155,6 +155,7 @@ const splitValues = (value) => value ? value.split(",") : []; const tasks = taskRows.map(([ task, rl, physics, renderer, presets, agentPresetCompatibility = {}, previewImage = "", supportsWarpFrontend = false, + pretrainedCheckpointPresetCompatibility = {}, defaultAlgorithms = {}, ]) => ({ task, @@ -166,6 +167,7 @@ agentPresetCompatibility, previewImage, supportsWarpFrontend, + pretrainedCheckpointPresetCompatibility, defaultAlgorithms, })); @@ -293,7 +295,14 @@ modeButton.setAttribute("aria-pressed", String(isActive)); } nonRlNote.hidden = supportsRl; - const supportsPretrainedCheckpoint = supportsRl && state.scope === "core"; + const task = selectedTask(); + const selectedPreset = fields.presets.value; + const compatiblePresets = [ + ...(task.pretrainedCheckpointPresetCompatibility["*"] || []), + ...(task.pretrainedCheckpointPresetCompatibility[fields.rl.value] || []), + ]; + const supportsPretrainedCheckpoint = supportsRl && state.scope === "core" + && (!selectedPreset || compatiblePresets.includes(selectedPreset)); fields.checkpoint.disabled = !supportsPretrainedCheckpoint; if (!supportsPretrainedCheckpoint) { fields.checkpoint.checked = false; @@ -676,6 +685,7 @@ }); for (const field of [fields.rl, fields.physics, fields.renderer, fields.presets, fields.checkpoint]) { field.addEventListener("change", () => { + updateModeControls(); commandOutput.textContent = currentCommand(); updatePreview(); }); diff --git a/scripts/tools/test/test_train_and_publish_checkpoints.py b/scripts/tools/test/test_train_and_publish_checkpoints.py index 9f263bba3298..01f59eb4fb6c 100644 --- a/scripts/tools/test/test_train_and_publish_checkpoints.py +++ b/scripts/tools/test/test_train_and_publish_checkpoints.py @@ -57,6 +57,7 @@ def test_job_commands_use_uv_run_isaaclab() -> None: task_name="Isaac-Test", physics_backend="physx", render_backend="none", + preset_names=("depth",), physics_selector="isaacsim_physx", ) args = Namespace(max_iterations=None, num_envs=None) @@ -66,8 +67,33 @@ def test_job_commands_use_uv_run_isaaclab() -> None: assert train_command[:4] == ["uv", "run", "isaaclab", "train"] assert play_command[:4] == ["uv", "run", "isaaclab", "play"] - assert train_command[-1] == "physics=isaacsim_physx" - assert play_command[-1] == "physics=isaacsim_physx" + assert train_command[-2:] == ["physics=isaacsim_physx", "presets=depth"] + assert play_command[-2:] == ["physics=isaacsim_physx", "presets=depth"] + + +def test_build_core_jobs_includes_declared_checkpoint_presets(monkeypatch: pytest.MonkeyPatch) -> None: + """Core jobs must include preset-specific checkpoints declared by the task.""" + task_spec = SimpleNamespace( + id="Isaac-Test", + kwargs={ + "env_cfg_entry_point": "isaaclab_tasks.core.test:TestEnvCfg", + "rl_games_cfg_entry_point": "isaaclab_tasks.core.test:TestAgentCfg", + "rsl_rl_cfg_entry_point": "isaaclab_tasks.core.test:TestAgentCfg", + "pretrained_checkpoint_preset_compatibility": {"rl_games": ("depth",)}, + }, + ) + monkeypatch.setattr("scripts.tools.train_and_publish_checkpoints.gym.registry", {task_spec.id: task_spec}) + monkeypatch.setattr("scripts.tools.train_and_publish_checkpoints.parse_env_cfg", lambda _: object()) + monkeypatch.setattr("scripts.tools.train_and_publish_checkpoints.enumerate_task_presets", lambda _: {}) + monkeypatch.setattr( + "scripts.tools.train_and_publish_checkpoints.get_pretrained_checkpoint_backend_names", + lambda _: ("physx", "rtx"), + ) + args = Namespace(physics_backends="physx", render_backends="rtx") + + jobs = _build_core_jobs(args) + + assert [(job.workflow, job.preset_names) for job in jobs] == [("rsl_rl", ()), ("rl_games", ("depth",))] def test_select_physics_variants_uses_concrete_isaac_sim_physx() -> None: @@ -130,8 +156,9 @@ def test_publish_uses_collected_checkpoint_without_training_logs( task_name="Isaac-Test", physics_backend="newtonmjwarp", render_backend="none", + preset_names=("depth",), ) - collected_path = tmp_path / "rsl_rl" / "Isaac-Test_newtonmjwarp_none_rsl_rl.pt" + collected_path = tmp_path / "rsl_rl" / "Isaac-Test_depth_newtonmjwarp_none_rsl_rl.pt" collected_path.parent.mkdir() collected_path.touch() args = Namespace( @@ -143,6 +170,6 @@ def test_publish_uses_collected_checkpoint_without_training_logs( assert publish_pretrained_checkpoint(job, args) assert ( - f"Publishing {collected_path} -> omniverse://checkpoints/rsl_rl/Isaac-Test_newtonmjwarp_none_rsl_rl.pt" + f"Publishing {collected_path} -> omniverse://checkpoints/rsl_rl/Isaac-Test_depth_newtonmjwarp_none_rsl_rl.pt" in capsys.readouterr().out ) diff --git a/scripts/tools/train_and_publish_checkpoints.py b/scripts/tools/train_and_publish_checkpoints.py index eddd6c43c018..e9b1cf695128 100644 --- a/scripts/tools/train_and_publish_checkpoints.py +++ b/scripts/tools/train_and_publish_checkpoints.py @@ -17,7 +17,7 @@ └── skrl/ Each checkpoint is named -``___``. +``[_]___``. State-only tasks use ``none`` as the render backend because their policies do not depend on rendering. This workflow targets core tasks only; other registered tasks do not receive published checkpoints from this matrix. The @@ -106,12 +106,13 @@ @dataclass(frozen=True) class CheckpointJob: - """One workflow, task, physics, and renderer training combination.""" + """One workflow, task, preset, physics, and renderer training combination.""" workflow: str task_name: str physics_backend: str | None = None render_backend: str | None = None + preset_names: tuple[str, ...] = () physics_selector: str | None = None render_selector: str | None = None agent: str | None = None @@ -122,7 +123,8 @@ def job_id(self) -> str: """Return the stable human-readable job identifier.""" if self.physics_backend is None: return f"{self.workflow}:{self.task_name}" - return f"{self.workflow}:{self.task_name}:{self.physics_backend}:{self.render_backend}" + parts = [self.workflow, self.task_name, *self.preset_names, self.physics_backend, self.render_backend] + return ":".join(parts) @property def experiment_name(self) -> str: @@ -134,18 +136,21 @@ def experiment_name(self) -> str: self.task_name, self.physics_backend, self.render_backend, + preset_names=self.preset_names, ) extension = os.path.splitext(filename)[1] return filename.removesuffix(extension) @property def preset_args(self) -> list[str]: - """Return typed preset selectors for this job.""" + """Return preset selectors for this job.""" args = [] if self.physics_selector is not None: args.append(f"physics={self.physics_selector}") if self.render_selector is not None: args.append(f"renderer={self.render_selector}") + if self.preset_names: + args.append(f"presets={','.join(self.preset_names)}") return args @@ -158,10 +163,7 @@ def _create_parser() -> argparse.ArgumentParser: parser.add_argument( "jobs", nargs="*", - help=( - "Job patterns. Legacy jobs use workflow:task. Core matrix patterns " - "also match workflow:task:physics:renderer." - ), + help="Job patterns. Legacy jobs use workflow:task. Core matrix patterns match the displayed job IDs.", ) parser.add_argument("-t", "--train", action="store_true", help="Run full training and collect checkpoints.") parser.add_argument("--smoke", action="store_true", help="Run one-iteration backend smoke training.") @@ -335,7 +337,14 @@ def _build_core_jobs(args: argparse.Namespace) -> list[CheckpointJob]: physics_variants = preset_map.get(PresetTarget.PHYSICS, []) render_variants = preset_map.get(PresetTarget.RENDERER, []) env_cfg = parse_env_cfg(task_spec.id) - workflow, agent, algorithm = _select_workflow(task_spec, env_cfg) + preferred_workflow = _select_workflow(task_spec, env_cfg) + checkpoint_compatibility = task_spec.kwargs.get("pretrained_checkpoint_preset_compatibility", {}) + workflow_selections = [preferred_workflow] + workflow_selections.extend( + (workflow, None, None) + for workflow in checkpoint_compatibility + if workflow != preferred_workflow[0] and f"{workflow}_cfg_entry_point" in task_spec.kwargs + ) default_physics = None if not physics_variants: default_physics, _ = get_pretrained_checkpoint_backend_names(env_cfg) @@ -347,21 +356,26 @@ def _build_core_jobs(args: argparse.Namespace) -> list[CheckpointJob]: physics_backends, ) render_selections = _select_render_variants(render_variants, render_backends) - for _physics_family, physics_selector in physics_selections: - physics_backend = _resolve_physics_backend(task_spec.id, physics_selector, default_physics) - for render_backend, render_selector in render_selections: - jobs.append( - CheckpointJob( - workflow=workflow, - task_name=task_spec.id, - physics_backend=physics_backend, - render_backend=render_backend, - physics_selector=physics_selector, - render_selector=render_selector, - agent=agent, - algorithm=algorithm, - ) - ) + for workflow, agent, algorithm in workflow_selections: + checkpoint_presets = [()] if workflow == preferred_workflow[0] else [] + checkpoint_presets.extend((preset_name,) for preset_name in checkpoint_compatibility.get(workflow, ())) + for _physics_family, physics_selector in physics_selections: + physics_backend = _resolve_physics_backend(task_spec.id, physics_selector, default_physics) + for render_backend, render_selector in render_selections: + for preset_names in checkpoint_presets: + jobs.append( + CheckpointJob( + workflow=workflow, + task_name=task_spec.id, + physics_backend=physics_backend, + render_backend=render_backend, + preset_names=preset_names, + physics_selector=physics_selector, + render_selector=render_selector, + agent=agent, + algorithm=algorithm, + ) + ) return jobs @@ -470,6 +484,7 @@ def _has_training_job_completed(job: CheckpointJob) -> bool: job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) if run_path is None or not os.path.isfile(os.path.join(run_path, _TRAINING_COMPLETE_FILENAME)): return False @@ -478,6 +493,7 @@ def _has_training_job_completed(job: CheckpointJob) -> bool: job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) @@ -488,6 +504,7 @@ def _mark_training_job_completed(job: CheckpointJob) -> None: job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) if run_path is None: raise RuntimeError(f"Unable to determine the latest run for {job.job_id}") @@ -513,6 +530,7 @@ def train_job(job: CheckpointJob, args: argparse.Namespace, smoke: bool = False) job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ): print(f"Training did not produce a checkpoint for {job.job_id}", file=sys.stderr) return False @@ -532,6 +550,7 @@ def collect_pretrained_checkpoint(job: CheckpointJob, output_dir: str, dry_run: job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) if source_path is None or not os.path.isfile(source_path): print(f"No completed checkpoint to collect for {job.job_id}") @@ -550,6 +569,7 @@ def review_pretrained_checkpoint(job: CheckpointJob, args: argparse.Namespace) - job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ): print(f"Skipping review of {job.job_id}; it has not been trained") return False @@ -558,6 +578,7 @@ def review_pretrained_checkpoint(job: CheckpointJob, args: argparse.Namespace) - job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ): print(f"Skipping review of {job.job_id}; training is incomplete") return False @@ -567,6 +588,7 @@ def review_pretrained_checkpoint(job: CheckpointJob, args: argparse.Namespace) - job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) if not args.force_review and review and review.get("reviewed"): print(f"Review already complete for {job.job_id}") @@ -577,6 +599,7 @@ def review_pretrained_checkpoint(job: CheckpointJob, args: argparse.Namespace) - job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) if checkpoint_path is None: print(f"Skipping review of {job.job_id}; no checkpoint was found") @@ -600,6 +623,7 @@ def review_pretrained_checkpoint(job: CheckpointJob, args: argparse.Namespace) - job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) if review_path is None: raise RuntimeError(f"Unable to determine review path for {job.job_id}") @@ -623,6 +647,7 @@ def publish_pretrained_checkpoint(job: CheckpointJob, args: argparse.Namespace) job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) if not review or review.get("result") != "accepted": print(f"Skipping publish of {job.job_id}; it does not have an accepted review") @@ -634,6 +659,7 @@ def publish_pretrained_checkpoint(job: CheckpointJob, args: argparse.Namespace) job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) else: filename = get_pretrained_checkpoint_filename( @@ -641,6 +667,7 @@ def publish_pretrained_checkpoint(job: CheckpointJob, args: argparse.Namespace) job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) publish_path = posixpath.join(args.publish_root.rstrip("/"), job.workflow, filename) print(f"Publishing {local_path} -> {publish_path}") @@ -664,6 +691,7 @@ def _summary_row(job: CheckpointJob, output_dir: str) -> list[str | bool]: job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) has_finished = ( _has_training_job_completed(job) @@ -676,10 +704,12 @@ def _summary_row(job: CheckpointJob, output_dir: str) -> list[str | bool]: job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) return [ job.workflow, job.task_name, + ",".join(job.preset_names), job.physics_backend or "", job.render_backend or "", job.physics_selector or "", @@ -698,6 +728,7 @@ def _get_collected_checkpoint_path(job: CheckpointJob, output_dir: str) -> str: job.task_name, job.physics_backend, job.render_backend, + preset_names=job.preset_names, ) path_parts = [output_dir, job.workflow] if job.physics_backend is None: @@ -733,6 +764,7 @@ def main(argv: list[str] | None = None) -> int: [ "Workflow", "Task", + "Presets", "Physics", "Renderer", "Physics selector", diff --git a/source/isaaclab_rl/changelog.d/mustafa-pretrained-checkpoint-presets.rst b/source/isaaclab_rl/changelog.d/mustafa-pretrained-checkpoint-presets.rst new file mode 100644 index 000000000000..e9626aa27569 --- /dev/null +++ b/source/isaaclab_rl/changelog.d/mustafa-pretrained-checkpoint-presets.rst @@ -0,0 +1,6 @@ +Fixed +^^^^^ + +* Fixed published checkpoint lookup ignoring non-default domain presets, which could fetch an + incompatible policy or miss an available preset-specific checkpoint. Preset-specific checkpoints + can now also be trained, collected, reviewed, and published through the checkpoint management tool. diff --git a/source/isaaclab_rl/isaaclab_rl/utils/pretrained_checkpoint.py b/source/isaaclab_rl/isaaclab_rl/utils/pretrained_checkpoint.py index becbc0e6feee..d64c31ff9f80 100644 --- a/source/isaaclab_rl/isaaclab_rl/utils/pretrained_checkpoint.py +++ b/source/isaaclab_rl/isaaclab_rl/utils/pretrained_checkpoint.py @@ -12,6 +12,8 @@ import json import os import posixpath +import sys +from collections.abc import Sequence from isaaclab.envs import DirectMARLEnvCfg, DirectRLEnvCfg, ManagerBasedRLEnvCfg from isaaclab.physics import PhysicsCfg @@ -73,11 +75,13 @@ def get_pretrained_checkpoint_filename( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] = (), ) -> str: """Return the published checkpoint filename. Backend-aware checkpoints use - ``___``. + ``[_]___``. Omitting both backend names returns the legacy workflow-specific filename. Args: @@ -86,6 +90,7 @@ def get_pretrained_checkpoint_filename( physics_backend: Physics backend name, such as ``"physx"``, ``"newtonmjwarp"``, or ``"newtonmjwarpvbdproxy"`` for a coupled solver. render_backend: Render backend name, such as ``"rtx"``, ``"newton"``, or ``"none"``. + preset_names: Non-default domain presets that affect policy compatibility. Returns: The checkpoint filename. @@ -95,7 +100,10 @@ def get_pretrained_checkpoint_filename( """ if workflow not in WORKFLOW_PRETRAINED_CHECKPOINT_EXTENSIONS: raise ValueError(f"Unsupported workflow: {workflow!r}") + preset_names = _normalize_pretrained_checkpoint_preset_names(preset_names) if physics_backend is None and render_backend is None: + if preset_names: + raise ValueError("preset_names require backend-aware checkpoint naming") return WORKFLOW_PRETRAINED_CHECKPOINT_FILENAMES[workflow] if physics_backend is None or render_backend is None: raise ValueError("physics_backend and render_backend must be provided together") @@ -103,12 +111,60 @@ def get_pretrained_checkpoint_filename( raise ValueError(f"Unsupported physics backend: {physics_backend!r}") if render_backend not in {"newton", "none", "rtx"}: raise ValueError(f"Unsupported render backend: {render_backend!r}") + preset_suffix = "" if not preset_names else f"_{'_'.join(preset_names)}" return ( - f"{task_name}_{physics_backend}_{render_backend}_{workflow}" + f"{task_name}{preset_suffix}_{physics_backend}_{render_backend}_{workflow}" f"{WORKFLOW_PRETRAINED_CHECKPOINT_EXTENSIONS[workflow]}" ) +def get_pretrained_checkpoint_preset_names(task_name: str, overrides: Sequence[str] | None = None) -> tuple[str, ...]: + """Return non-default domain presets selected for a checkpoint. + + Preset aliases whose value is the preset's ``default`` are omitted so existing + unsuffixed checkpoints remain compatible. Typed physics and renderer selectors + are also omitted because they already have dedicated checkpoint fields. + + Args: + task_name: Registered task name. + overrides: Hydra-style overrides. Defaults to :data:`sys.argv`. + + Returns: + Selected non-default domain preset names in canonical order. + """ + from isaaclab_tasks.utils.hydra import collect_presets + from isaaclab_tasks.utils.preset_target import PresetTarget + + selected_names = [] + for override in sys.argv[1:] if overrides is None else overrides: + if "=" not in override: + continue + key, value = override.split("=", 1) + if key.lstrip("-") == PresetTarget.DOMAIN.value: + selected_names.extend(name.strip() for name in value.split(",") if name.strip()) + + if not selected_names: + return () + + preset_fields = collect_presets(load_cfg_from_registry(task_name, "env_cfg_entry_point")) + typed_targets = tuple(target for target in PresetTarget if target.base_classes) + typed_names = { + name + for fields in preset_fields.values() + for name, value in fields.items() + if any(target.matches(value) for target in typed_targets) + } + non_default_domain_names = { + name + for fields in preset_fields.values() + for name, value in fields.items() + if name != "default" and name not in typed_names and not _preset_value_matches_default(value, fields) + } + return _normalize_pretrained_checkpoint_preset_names( + name for name in selected_names if name in non_default_domain_names + ) + + def get_pretrained_checkpoint_backend_names( env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, ) -> tuple[str, str]: @@ -145,9 +201,13 @@ def get_log_root_path( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] = (), ) -> str: """Return the absolute log root for a workflow, task, and backend combination.""" - experiment_name = _get_pretrained_checkpoint_stem(workflow, task_name, physics_backend, render_backend) + experiment_name = _get_pretrained_checkpoint_stem( + workflow, task_name, physics_backend, render_backend, preset_names=preset_names + ) return os.path.abspath(os.path.join("logs", workflow, experiment_name)) @@ -156,9 +216,11 @@ def get_latest_job_run_path( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] = (), ) -> str | None: """Return the local log path of the most recent matching run.""" - log_root_path = get_log_root_path(workflow, task_name, physics_backend, render_backend) + log_root_path = get_log_root_path(workflow, task_name, physics_backend, render_backend, preset_names=preset_names) return _get_latest_file_or_directory(log_root_path) @@ -167,13 +229,17 @@ def get_pretrained_checkpoint_path( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] = (), ) -> str | None: """Return the trained checkpoint path from the latest local run.""" - path = get_latest_job_run_path(workflow, task_name, physics_backend, render_backend) + path = get_latest_job_run_path(workflow, task_name, physics_backend, render_backend, preset_names=preset_names) if not path: return None - checkpoint_stem = _get_pretrained_checkpoint_stem(workflow, task_name, physics_backend, render_backend) + checkpoint_stem = _get_pretrained_checkpoint_stem( + workflow, task_name, physics_backend, render_backend, preset_names=preset_names + ) if workflow == "rl_games": preferred_path = os.path.join(path, "nn", f"{checkpoint_stem}.pth") if os.path.isfile(preferred_path): @@ -197,9 +263,13 @@ def get_pretrained_checkpoint_publish_path( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] = (), ) -> str: """Return the path where a checkpoint is published.""" - filename = get_pretrained_checkpoint_filename(workflow, task_name, physics_backend, render_backend) + filename = get_pretrained_checkpoint_filename( + workflow, task_name, physics_backend, render_backend, preset_names=preset_names + ) if physics_backend is None: return posixpath.join(PRETRAINED_CHECKPOINT_PATH, workflow, task_name, filename) return posixpath.join(PRETRAINED_CHECKPOINT_PATH, workflow, filename) @@ -210,9 +280,13 @@ def get_published_pretrained_checkpoint_path( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] = (), ) -> str: """Return the path from which a published checkpoint is fetched.""" - filename = get_pretrained_checkpoint_filename(workflow, task_name, physics_backend, render_backend) + filename = get_pretrained_checkpoint_filename( + workflow, task_name, physics_backend, render_backend, preset_names=preset_names + ) path_parts = [ISAACLAB_NUCLEUS_DIR, "PretrainedCheckpoints", workflow] if physics_backend is None: path_parts.append(task_name) @@ -224,6 +298,8 @@ def get_published_pretrained_checkpoint( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] | None = None, ) -> str | None: """Gets the path for the pre-trained checkpoint. @@ -237,6 +313,9 @@ def get_published_pretrained_checkpoint( to use the legacy checkpoint layout. render_backend: Render backend name. Omit with :paramref:`physics_backend` to use the legacy checkpoint layout. + preset_names: Non-default domain presets that affect policy compatibility. + For backend-aware checkpoints, defaults to resolving ``presets=`` selectors + from :data:`sys.argv`. Legacy checkpoints do not use preset-qualified names. Returns: The path, or None when the asset server does not report a checkpoint for this task @@ -250,8 +329,16 @@ def get_published_pretrained_checkpoint( instance because the local cache directory is not writable. The originating error is chained as the cause. """ - filename = get_pretrained_checkpoint_filename(workflow, task_name, physics_backend, render_backend) - ov_path = get_published_pretrained_checkpoint_path(workflow, task_name, physics_backend, render_backend) + if preset_names is None and physics_backend is not None and render_backend is not None: + preset_names = get_pretrained_checkpoint_preset_names(task_name) + elif preset_names is None: + preset_names = () + filename = get_pretrained_checkpoint_filename( + workflow, task_name, physics_backend, render_backend, preset_names=preset_names + ) + ov_path = get_published_pretrained_checkpoint_path( + workflow, task_name, physics_backend, render_backend, preset_names=preset_names + ) download_dir = os.path.join(".pretrained_checkpoints", workflow) if physics_backend is None: download_dir = os.path.join(download_dir, task_name) @@ -300,9 +387,13 @@ def has_pretrained_checkpoint_job_run( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] = (), ) -> bool: """Return whether an experiment exists for the workflow, task, and backends.""" - return os.path.exists(get_log_root_path(workflow, task_name, physics_backend, render_backend)) + return os.path.exists( + get_log_root_path(workflow, task_name, physics_backend, render_backend, preset_names=preset_names) + ) def has_pretrained_checkpoint_job_finished( @@ -310,9 +401,13 @@ def has_pretrained_checkpoint_job_finished( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] = (), ) -> bool: """Return whether an experiment has a checkpoint result.""" - local_path = get_pretrained_checkpoint_path(workflow, task_name, physics_backend, render_backend) + local_path = get_pretrained_checkpoint_path( + workflow, task_name, physics_backend, render_backend, preset_names=preset_names + ) return local_path is not None and os.path.exists(local_path) @@ -321,9 +416,11 @@ def get_pretrained_checkpoint_review_path( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] = (), ) -> str | None: """Return the review JSON path for a workflow, task, and backends.""" - run_path = get_latest_job_run_path(workflow, task_name, physics_backend, render_backend) + run_path = get_latest_job_run_path(workflow, task_name, physics_backend, render_backend, preset_names=preset_names) if not run_path: return None return os.path.join(run_path, "pretrained_checkpoint_review.json") @@ -334,9 +431,13 @@ def get_pretrained_checkpoint_review( task_name: str, physics_backend: str | None = None, render_backend: str | None = None, + *, + preset_names: Sequence[str] = (), ) -> dict | None: """Return the review JSON data for a workflow, task, and backends.""" - review_path = get_pretrained_checkpoint_review_path(workflow, task_name, physics_backend, render_backend) + review_path = get_pretrained_checkpoint_review_path( + workflow, task_name, physics_backend, render_backend, preset_names=preset_names + ) if not review_path: return None @@ -422,14 +523,36 @@ def _get_pretrained_checkpoint_stem( task_name: str, physics_backend: str | None, render_backend: str | None, + *, + preset_names: Sequence[str] = (), ) -> str: """Return the checkpoint filename without its workflow extension.""" if physics_backend is None and render_backend is None: return task_name - filename = get_pretrained_checkpoint_filename(workflow, task_name, physics_backend, render_backend) + filename = get_pretrained_checkpoint_filename( + workflow, task_name, physics_backend, render_backend, preset_names=preset_names + ) return filename.removesuffix(WORKFLOW_PRETRAINED_CHECKPOINT_EXTENSIONS[workflow]) +def _normalize_pretrained_checkpoint_preset_names(preset_names: Sequence[str]) -> tuple[str, ...]: + """Return unique, validated checkpoint preset names in canonical order.""" + normalized = tuple(sorted(set(preset_names))) + invalid = [name for name in normalized if not name or not name.replace("_", "").isalnum()] + if invalid: + raise ValueError(f"Invalid checkpoint preset names: {invalid}") + return normalized + + +def _preset_value_matches_default(value, fields: dict) -> bool: + """Return whether a preset value is structurally equivalent to its default.""" + default = fields.get("default") + try: + return bool(value == default) + except (RuntimeError, TypeError, ValueError): + return value is default + + def _get_latest_file_or_directory(path: str, pattern: str = "*") -> str | None: """Returns the path to the most recently modified file or directory at a path matching an optional pattern""" g = glob.glob(f"{path}/{pattern}") diff --git a/source/isaaclab_rl/test/test_pretrained_checkpoint.py b/source/isaaclab_rl/test/test_pretrained_checkpoint.py index 32ff4148d951..b1789839ec67 100644 --- a/source/isaaclab_rl/test/test_pretrained_checkpoint.py +++ b/source/isaaclab_rl/test/test_pretrained_checkpoint.py @@ -47,9 +47,33 @@ def test_get_pretrained_checkpoint_filename_includes_backends(): assert filename == "Isaac-Cartpole_newtonmjwarp_rtx_rsl_rl.pt" -def test_get_pretrained_checkpoint_filename_preserves_legacy_layout(): +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + (("presets=depth",), ("depth",)), + (("presets=rgb",), ()), + (("physics=newton_mjwarp", "renderer=newton_renderer"), ()), + ], +) +def test_get_pretrained_checkpoint_preset_names_uses_non_default_domain_presets(overrides, expected): + """Test that default aliases and typed backends do not duplicate checkpoint identity fields.""" + preset_names = pretrained_checkpoint.get_pretrained_checkpoint_preset_names( + "Isaac-Cartpole-Camera-Direct", overrides + ) + + assert preset_names == expected + + +def test_get_pretrained_checkpoint_filename_preserves_legacy_layout(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): """Test that callers omitting both backends retain the legacy filename.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("sys.argv", ["play.py", "presets=depth"]) + cached_path = Path(".pretrained_checkpoints/rl_games/Isaac-Cartpole/checkpoint.pth") + cached_path.parent.mkdir(parents=True) + cached_path.touch() + assert pretrained_checkpoint.get_pretrained_checkpoint_filename("rl_games", "Isaac-Cartpole") == "checkpoint.pth" + assert pretrained_checkpoint.get_published_pretrained_checkpoint("rl_games", "Isaac-Cartpole") == str(cached_path) def test_get_log_root_path_preserves_legacy_task_name(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): @@ -149,6 +173,7 @@ def test_get_published_pretrained_checkpoint_downloads_to_flat_cache( """Test that backend-aware downloads use the workflow cache directory.""" monkeypatch.chdir(tmp_path) monkeypatch.setattr(pretrained_checkpoint, "ISAACLAB_NUCLEUS_DIR", "omniverse://IsaacLab") + monkeypatch.setattr("sys.argv", ["play.py", "presets=depth"]) retrieved = {} def _retrieve_file_path(remote_path: str, download_dir: str) -> str: @@ -162,16 +187,17 @@ def _retrieve_file_path(remote_path: str, download_dir: str) -> str: monkeypatch.setattr(pretrained_checkpoint, "retrieve_file_path", _retrieve_file_path) path = pretrained_checkpoint.get_published_pretrained_checkpoint( - "rsl_rl", - "Isaac-Cartpole", - "physx", - "none", + "rl_games", + "Isaac-Cartpole-Camera-Direct", + "newtonmjwarp", + "newton", ) - expected_download_dir = str(Path(".pretrained_checkpoints") / "rsl_rl") - assert path == str(Path(expected_download_dir) / "Isaac-Cartpole_physx_none_rsl_rl.pt") + expected_download_dir = str(Path(".pretrained_checkpoints") / "rl_games") + filename = "Isaac-Cartpole-Camera-Direct_depth_newtonmjwarp_newton_rl_games.pth" + assert path == str(Path(expected_download_dir) / filename) assert retrieved == { - "remote_path": "omniverse://IsaacLab/PretrainedCheckpoints/rsl_rl/Isaac-Cartpole_physx_none_rsl_rl.pt", + "remote_path": f"omniverse://IsaacLab/PretrainedCheckpoints/rl_games/{filename}", "download_dir": expected_download_dir, } diff --git a/source/isaaclab_tasks/changelog.d/mustafa-pretrained-checkpoint-presets.skip b/source/isaaclab_tasks/changelog.d/mustafa-pretrained-checkpoint-presets.skip new file mode 100644 index 000000000000..129cb0981603 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/mustafa-pretrained-checkpoint-presets.skip @@ -0,0 +1 @@ +Declared preset-specific checkpoint availability for the environment browser. diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/__init__.py b/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/__init__.py index 064f647ecf83..e914f87afbc7 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/__init__.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/__init__.py @@ -53,6 +53,7 @@ "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:CartpoleCameraDirectPPORunnerCfg", "default_agent": "rsl_rl", "skrl_cfg_entry_point": f"{agents.__name__}:skrl_direct_camera_ppo_cfg.yaml", + "pretrained_checkpoint_preset_compatibility": {"rl_games": ("depth",)}, }, ) diff --git a/tools/environ_docs.py b/tools/environ_docs.py index 28d21be751d8..8aecb6da2de7 100644 --- a/tools/environ_docs.py +++ b/tools/environ_docs.py @@ -125,6 +125,7 @@ class EnvironmentDocRow: presets: dict[PresetTarget, list[str]] | None agent_preset_compatibility: dict[str, tuple[str, ...]] = field(default_factory=dict) supports_warp_frontend: bool = False + pretrained_checkpoint_preset_compatibility: dict[str, tuple[str, ...]] = field(default_factory=dict) def _supports_warp_frontend(task_name: str, workflow: str, presets: dict[PresetTarget, list[str]] | None) -> bool: @@ -288,6 +289,27 @@ def _domain_presets_for_docs(preset_map: dict[PresetTarget, list[str]]) -> list[ return domain_names +def _default_domain_presets(task_name: str) -> tuple[str, ...]: + """Return domain preset aliases that resolve to the task's default config.""" + from isaaclab_tasks.utils.hydra import collect_presets + from isaaclab_tasks.utils.parse_cfg import load_cfg_from_registry + + fields_by_path = collect_presets(load_cfg_from_registry(task_name, "env_cfg_entry_point")) + typed_targets = tuple(target for target in PresetTarget if target.base_classes) + aliases: dict[str, bool] = {} + for fields in fields_by_path.values(): + default = fields.get("default") + for name, value in fields.items(): + if name == "default" or any(target.matches(value) for target in typed_targets): + continue + try: + matches_default = bool(value == default) + except (RuntimeError, TypeError, ValueError): + matches_default = value is default + aliases[name] = aliases.get(name, True) and matches_default + return tuple(sorted(name for name, matches_default in aliases.items() if matches_default)) + + def _selector_names_for_docs( preset_map: dict[PresetTarget, list[str]] | None, ) -> dict[PresetTarget, list[str]]: @@ -581,6 +603,20 @@ def collect_environment_doc_rows( preset_map[PresetTarget.PHYSICS] = _physics_names_for_docs(spec.id, preset_map) preset_map = _apply_preset_exclusions(spec.id, preset_map) agents = apply_rl_library_overrides(spec.id, parse_rl_libraries_from_kwargs(spec.kwargs)) + visible_domain_presets = set(_selector_names_for_docs(preset_map)[PresetTarget.DOMAIN]) + default_checkpoint_presets = () + if spec.id.startswith("Isaac-"): + with contextlib.suppress(Exception): + default_checkpoint_presets = tuple( + name for name in _default_domain_presets(spec.id) if name in visible_domain_presets + ) + checkpoint_preset_compatibility = { + library: tuple(preset for preset in presets if preset in visible_domain_presets) + for library, presets in spec.kwargs.get("pretrained_checkpoint_preset_compatibility", {}).items() + if library in agents + } + if default_checkpoint_presets: + checkpoint_preset_compatibility["*"] = default_checkpoint_presets workflow = get_workflow(spec.entry_point) rows.append( @@ -595,6 +631,7 @@ def collect_environment_doc_rows( if agent in spec.kwargs }, supports_warp_frontend=_supports_warp_frontend(spec.id, workflow, preset_map), + pretrained_checkpoint_preset_compatibility=checkpoint_preset_compatibility, ) ) @@ -680,14 +717,24 @@ def render_environment_browser_task_rows( if aliases: preview_image = max(aliases, key=lambda item: len(item[0]))[1] default_algorithms = {"skrl": "MAPPO"} if "MAPPO" in row.rl_libraries.get("skrl", []) else {} - if row.agent_preset_compatibility or preview_image or row.supports_warp_frontend or default_algorithms: - rendered_values += f", {json.dumps(row.agent_preset_compatibility, sort_keys=True)}" - if preview_image or row.supports_warp_frontend or default_algorithms: - rendered_values += f", {json.dumps(preview_image)}" - if row.supports_warp_frontend or default_algorithms: - rendered_values += f", {json.dumps(row.supports_warp_frontend)}" - if default_algorithms: - rendered_values += f", {json.dumps(default_algorithms, sort_keys=True)}" + optional_values = [ + row.agent_preset_compatibility, + preview_image, + row.supports_warp_frontend, + row.pretrained_checkpoint_preset_compatibility, + default_algorithms, + ] + optional_defaults = [{}, "", False, {}, {}] + last_value = next( + ( + index + for index in reversed(range(len(optional_values))) + if optional_values[index] != optional_defaults[index] + ), + -1, + ) + for value in optional_values[: last_value + 1]: + rendered_values += f", {json.dumps(value, sort_keys=True)}" lines.append(f" [{rendered_values}],") lines.append(" ];") return "\n".join(lines) diff --git a/tools/test/test_environ_docs.py b/tools/test/test_environ_docs.py index 1bcb6531e3cd..710a45a6f799 100644 --- a/tools/test/test_environ_docs.py +++ b/tools/test/test_environ_docs.py @@ -10,6 +10,7 @@ import sys from pathlib import Path +import gymnasium as gym import pytest from gymnasium.envs.registration import EnvSpec @@ -292,6 +293,16 @@ def test_collect_environment_doc_rows_includes_registered_agent_preset_compatibi } +def test_collect_environment_doc_rows_includes_checkpoint_preset_compatibility(): + """Checkpoint preset availability must be carried into generated browser rows.""" + row = collect_environment_doc_rows([gym.spec("Isaac-Cartpole-Camera-Direct")])[0] + + assert row.pretrained_checkpoint_preset_compatibility == { + "*": ("rgb",), + "rl_games": ("depth",), + } + + def test_collect_environment_doc_rows_excludes_deprecated_task_aliases(): specs = [ EnvSpec( @@ -423,6 +434,7 @@ def test_environment_browser_rows_include_concrete_core_and_contributed_selector "rsl_rl_feature_cfg_entry_point": ("resnet18", "theia_tiny"), }, supports_warp_frontend=True, + pretrained_checkpoint_preset_compatibility={"*": ("rgb",), "rsl_rl": ("depth",)}, ), EnvironmentDocRow( task_name="IsaacContrib-Cartpole", @@ -458,6 +470,8 @@ def test_environment_browser_rows_include_concrete_core_and_contributed_selector assert '"ovphysx"' in updated assert '"tasks/classic/cartpole.jpg"' in updated assert '"tasks/classic/cartpole.jpg", true' in updated + assert '"*": ["rgb"]' in updated + assert '"rsl_rl": ["depth"]' in updated assert updated.index('"Isaac-Cartpole"') < updated.index('"IsaacContrib-Cartpole"') assert "const preserved = true;" in updated @@ -475,7 +489,7 @@ def test_environment_browser_rows_include_mappo_as_the_skrl_default(): rendered = render_environment_browser_task_rows(rows) - assert '["Isaac-Multi-Agent-Direct", "skrl", "", "", "", {}, "", false, {"skrl": "MAPPO"}]' in rendered + assert '["Isaac-Multi-Agent-Direct", "skrl", "", "", "", {}, "", false, {}, {"skrl": "MAPPO"}]' in rendered def test_collect_environment_browser_preview_images_preserves_generated_assignments(): From f443e8ac57a70f1757546bae6e91cad74b150dc4 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Mon, 7 Sep 2026 21:12:53 -0700 Subject: [PATCH 016/128] Default surface-gripper tasks to CPU simulation (#7627) # Description Fixes NVBug 6684415. The three contrib stack tasks that configure a PhysX `SurfaceGripper` now select CPU simulation by default, and config validation rejects an explicit unsupported GPU override before simulator initialization. The zero and random agent entrypoints now preserve a task-defined simulation device unless the user explicitly passes `--device`, allowing `AppLauncher` to start these tasks on CPU automatically. The public `parse_env_cfg` helper also preserves the registered task device when its `device` argument is omitted, while continuing to apply explicit device overrides. This also corrects the two UR10 suction configs, which previously assigned `self.device` instead of `self.sim.device`. No new dependencies are required. ## Type of change - Bug fix (non-breaking change which fixes an issue) ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable. ## Testing - `uv run --isolated --frozen --extra test --extra teleop python -m pytest source/isaaclab_rl/test/test_entrypoints.py source/isaaclab_tasks/test/core/test_parse_cfg.py source/isaaclab_tasks/test/contrib/stack/test_surface_gripper_task_cfg.py -q` (`44 passed, 2 skipped`) - `uv run --frozen isaaclab -f` - `uv run --frozen python tools/changelog/cli.py check develop` - Verified the task-config regression test fails against the reported 3.0 commit because all three affected tasks resolved to `cuda:0`. - Verified the `parse_env_cfg` regression against the first PR revision: the omitted device resolved back to `cuda:0` and failed CPU-only validation. The full Isaac Sim smoke test was attempted in a fresh environment but stopped at the interactive Omniverse EULA prompt before Kit startup. ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `uv run isaaclab -f` - [x] I have confirmed that no documentation or environment-browser selector change is required - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package - [x] My name already exists in `CONTRIBUTORS.md` --- ...llyg-fix-galbot-surface-gripper-device.rst | 4 + .../isaaclab_rl/entrypoints/simple_agents.py | 9 ++- source/isaaclab_rl/test/test_entrypoints.py | 80 +++++++++++++++++++ ...llyg-fix-galbot-surface-gripper-device.rst | 6 ++ .../config/galbot/stack_joint_pos_env_cfg.py | 5 ++ .../ur10_gripper/stack_joint_pos_env_cfg.py | 6 +- .../contrib/stack/stack_env_cfg.py | 14 ++++ .../isaaclab_tasks/utils/parse_cfg.py | 7 +- .../stack/test_surface_gripper_task_cfg.py | 34 ++++++++ .../test/core/test_parse_cfg.py | 15 ++++ 10 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 source/isaaclab_rl/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst create mode 100644 source/isaaclab_tasks/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst create mode 100644 source/isaaclab_tasks/test/contrib/stack/test_surface_gripper_task_cfg.py diff --git a/source/isaaclab_rl/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst b/source/isaaclab_rl/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst new file mode 100644 index 000000000000..9fae38116873 --- /dev/null +++ b/source/isaaclab_rl/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst @@ -0,0 +1,4 @@ +Fixed +^^^^^ + +* Fixed the zero and random agents overriding task-defined simulation devices when ``--device`` was omitted. diff --git a/source/isaaclab_rl/isaaclab_rl/entrypoints/simple_agents.py b/source/isaaclab_rl/isaaclab_rl/entrypoints/simple_agents.py index 0f1ab877b42a..67576dd9f75b 100644 --- a/source/isaaclab_rl/isaaclab_rl/entrypoints/simple_agents.py +++ b/source/isaaclab_rl/isaaclab_rl/entrypoints/simple_agents.py @@ -67,7 +67,10 @@ def run(argv: list[str] | None = None, *, policy: PolicyName) -> None: # override with CLI arguments and reject unsupported configurations before # launching Kit or initializing a native physics backend. env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs - env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device + if args_cli.device is not None: + env_cfg.sim.device = args_cli.device + # Pass the resolved task device through to AppLauncher. + args_cli.device = env_cfg.sim.device if args_cli.disable_fabric: env_cfg.sim.use_fabric = False try: @@ -249,8 +252,8 @@ def _parse_args(argv: list[str] | None, policy: PolicyName) -> argparse.Namespac ) # append AppLauncher cli args add_launcher_args(parser) - # Keep checkpoint-free agents on the kitless default path. - parser.set_defaults(visualizer=["newton_gl"]) + # Let task configs select the simulation device and keep checkpoint-free agents on the kitless default path. + parser.set_defaults(device=None, visualizer=["newton_gl"]) args_cli, hydra_args = setup_preset_cli(parser, argv) sys.argv = [sys.argv[0]] + hydra_args return args_cli diff --git a/source/isaaclab_rl/test/test_entrypoints.py b/source/isaaclab_rl/test/test_entrypoints.py index c821acf61093..43d041062e74 100644 --- a/source/isaaclab_rl/test/test_entrypoints.py +++ b/source/isaaclab_rl/test/test_entrypoints.py @@ -234,9 +234,89 @@ def test_simple_agents_default_to_newton_visualizer( args = _simple_agents._parse_args([], policy) + assert args.device is None assert args.visualizer == ["newton_gl"] +@pytest.mark.parametrize("policy", ["zero", "random"]) +def test_simple_agents_accept_explicit_device( + policy: _simple_agents.PolicyName, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Checkpoint-free agents should retain an explicit CLI device.""" + monkeypatch.setattr(sys, "argv", ["pytest"]) + + args = _simple_agents._parse_args(["--device", "cuda:1"], policy) + + assert args.device == "cuda:1" + + +def test_simple_agents_preserve_task_device_default(monkeypatch: pytest.MonkeyPatch) -> None: + """Checkpoint-free agents should not replace a task-required device with the CLI default.""" + + class _ExpectedStop(Exception): + pass + + class _Cfg: + scene = SimpleNamespace(num_envs=1) + sim = SimpleNamespace(device="cpu", use_fabric=True) + + def validate(self) -> None: + pass + + args = SimpleNamespace( + num_envs=None, + device=None, + disable_fabric=False, + task="Cpu-Task", + ) + + def launch_simulation(cfg, launcher_args): + assert cfg.sim.device == "cpu" + assert launcher_args.device == "cpu" + raise _ExpectedStop + + monkeypatch.setattr(_simple_agents, "_parse_args", lambda argv, policy: args) + monkeypatch.setattr(_simple_agents, "resolve_task_config", lambda task, agent: (_Cfg(), None)) + monkeypatch.setattr(_simple_agents, "launch_simulation", launch_simulation) + + with pytest.raises(_ExpectedStop): + _simple_agents.run([], policy="zero") + + +def test_simple_agents_apply_explicit_device_override(monkeypatch: pytest.MonkeyPatch) -> None: + """Checkpoint-free agents should continue to honor an explicit CLI device.""" + + class _ExpectedStop(Exception): + pass + + class _Cfg: + scene = SimpleNamespace(num_envs=1) + sim = SimpleNamespace(device="cpu", use_fabric=True) + + def validate(self) -> None: + pass + + args = SimpleNamespace( + num_envs=None, + device="cuda:1", + disable_fabric=False, + task="Cpu-Task", + ) + + def launch_simulation(cfg, launcher_args): + assert cfg.sim.device == "cuda:1" + assert launcher_args.device == "cuda:1" + raise _ExpectedStop + + monkeypatch.setattr(_simple_agents, "_parse_args", lambda argv, policy: args) + monkeypatch.setattr(_simple_agents, "resolve_task_config", lambda task, agent: (_Cfg(), None)) + monkeypatch.setattr(_simple_agents, "launch_simulation", launch_simulation) + + with pytest.raises(_ExpectedStop): + _simple_agents.run([], policy="random") + + def test_zero_agent_rejects_invalid_config_before_launch(monkeypatch: pytest.MonkeyPatch) -> None: """Unsupported task presets fail cleanly before a simulator backend is initialized.""" diff --git a/source/isaaclab_tasks/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst b/source/isaaclab_tasks/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst new file mode 100644 index 000000000000..2cf490f53e0c --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst @@ -0,0 +1,6 @@ +Fixed +^^^^^ + +* Fixed surface-gripper stack tasks to select CPU simulation by default and reject unsupported GPU overrides before + simulator initialization. Task-defined simulation devices are now preserved by :func:`parse_env_cfg` when no + explicit device override is provided. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/galbot/stack_joint_pos_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/galbot/stack_joint_pos_env_cfg.py index 4f3f9a13fbd9..e6a329d0e916 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/galbot/stack_joint_pos_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/galbot/stack_joint_pos_env_cfg.py @@ -26,6 +26,7 @@ from isaaclab_tasks.contrib.stack.stack_env_cfg import ( ObservationsCfg, StackEnvCfg, + raise_if_surface_gripper_on_gpu, raise_if_surface_gripper_on_newton, ) @@ -348,11 +349,15 @@ class GalbotRightArmCubeStackEnvCfg(GalbotLeftArmCubeStackEnvCfg): def validate_config(self): # The right-arm suction cup uses a PhysX-only surface gripper. raise_if_surface_gripper_on_newton(self) + raise_if_surface_gripper_on_gpu(self) def __post_init__(self): # post init of parent super().__post_init__() + # Surface grippers currently require CPU simulation. + self.sim.device = "cpu" + # Move to area below right hand (invert y-axis) left, right = self.events.randomize_cube_positions.params["pose_range"]["y"] self.events.randomize_cube_positions.params["pose_range"]["y"] = (-right, -left) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/ur10_gripper/stack_joint_pos_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/ur10_gripper/stack_joint_pos_env_cfg.py index a561caac7429..22806ed3aed1 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/ur10_gripper/stack_joint_pos_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/ur10_gripper/stack_joint_pos_env_cfg.py @@ -20,6 +20,7 @@ from isaaclab_tasks.contrib.stack.mdp import franka_stack_events from isaaclab_tasks.contrib.stack.stack_env_cfg import ( StackEnvCfg, + raise_if_surface_gripper_on_gpu, raise_if_surface_gripper_on_newton, ) @@ -88,6 +89,7 @@ class UR10CubeStackEnvCfg(StackEnvCfg): def validate_config(self): # Surface grippers used by these suction robots are PhysX-only. raise_if_surface_gripper_on_newton(self) + raise_if_surface_gripper_on_gpu(self) def __post_init__(self): # post init of parent @@ -152,7 +154,7 @@ def __post_init__(self): super().__post_init__() # Suction grippers currently require CPU simulation - self.device = "cpu" + self.sim.device = "cpu" # Set events self.events = EventCfgLongSuction() @@ -192,7 +194,7 @@ def __post_init__(self): super().__post_init__() # Suction grippers currently require CPU simulation - self.device = "cpu" + self.sim.device = "cpu" # Set UR10 as robot self.scene.robot = UR10_SHORT_SUCTION_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/stack_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/stack_env_cfg.py index 80e30872bd0e..4f8602068e81 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/stack_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/stack_env_cfg.py @@ -342,6 +342,20 @@ def raise_if_surface_gripper_on_newton(env_cfg) -> None: ) +def raise_if_surface_gripper_on_gpu(env_cfg) -> None: + """Reject GPU simulation for scenes that configure a surface gripper. + + Args: + env_cfg: The resolved environment config to inspect. + """ + if getattr(env_cfg.scene, "surface_gripper", None) is None: + return + if env_cfg.sim.device != "cpu": + raise ValueError( + "Surface grippers are only supported on the CPU simulation device. Re-run this task with --device cpu." + ) + + @configclass class StackEnvCfg(ManagerBasedRLEnvCfg): """Configuration for the stacking environment.""" diff --git a/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py index 3b879e5c22c3..ec280218791f 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/utils/parse_cfg.py @@ -146,7 +146,7 @@ def load_cfg_from_registry(task_name: str, entry_point_key: str) -> dict | objec def parse_env_cfg( task_name: str, - device: str = "cuda:0", + device: str | None = None, num_envs: int | None = None, use_fabric: bool | None = None, overrides: Sequence[str] = (), @@ -155,7 +155,7 @@ def parse_env_cfg( Args: task_name: The name of the environment. - device: The device to run the simulation on. Defaults to "cuda:0". + device: The device to run the simulation on. Defaults to None, in which case it is left unchanged. num_envs: Number of environments to create. Defaults to None, in which case it is left unchanged. use_fabric: Whether to enable/disable fabric interface. If false, all read/write operations go through USD. This slows down the simulation but allows seeing the changes in the USD through the USD stage. @@ -190,7 +190,8 @@ def parse_env_cfg( raise RuntimeError(f"Configuration for the task: '{task_name}' is not a class. Please provide a class.") # simulation device - cfg.sim.device = device + if device is not None: + cfg.sim.device = device # disable fabric to read/write through USD if use_fabric is not None: cfg.sim.use_fabric = use_fabric diff --git a/source/isaaclab_tasks/test/contrib/stack/test_surface_gripper_task_cfg.py b/source/isaaclab_tasks/test/contrib/stack/test_surface_gripper_task_cfg.py new file mode 100644 index 000000000000..95c860ae9bb2 --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/stack/test_surface_gripper_task_cfg.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest + +import isaaclab_tasks # noqa: F401 +from isaaclab_tasks.utils.parse_cfg import load_cfg_from_registry + +_SURFACE_GRIPPER_TASKS = [ + "IsaacContrib-Stack-Cube-Galbot-Right-Arm-Suction-RmpFlow", + "IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel", + "IsaacContrib-Stack-Cube-UR10-Short-Suction-IK-Rel", +] + + +@pytest.mark.parametrize("task_name", _SURFACE_GRIPPER_TASKS) +def test_surface_gripper_tasks_default_to_cpu(task_name: str) -> None: + """Surface-gripper tasks should select their only supported simulation device.""" + env_cfg = load_cfg_from_registry(task_name, "env_cfg_entry_point") + + assert env_cfg.sim.device == "cpu" + env_cfg.validate() + + +@pytest.mark.parametrize("task_name", _SURFACE_GRIPPER_TASKS) +def test_surface_gripper_tasks_reject_gpu_override(task_name: str) -> None: + """An explicit unsupported GPU device should fail during config validation.""" + env_cfg = load_cfg_from_registry(task_name, "env_cfg_entry_point") + env_cfg.sim.device = "cuda:0" + + with pytest.raises(ValueError, match="only supported on the CPU simulation device"): + env_cfg.validate() diff --git a/source/isaaclab_tasks/test/core/test_parse_cfg.py b/source/isaaclab_tasks/test/core/test_parse_cfg.py index 7d4aa2fc1a99..3182767aa630 100644 --- a/source/isaaclab_tasks/test/core/test_parse_cfg.py +++ b/source/isaaclab_tasks/test/core/test_parse_cfg.py @@ -21,3 +21,18 @@ def test_parse_env_cfg_accepts_list_overrides(): """A properly wrapped override list should apply without error.""" env_cfg = parse_env_cfg("Isaac-Cartpole", overrides=["physics=isaacsim_physx"]) assert env_cfg is not None + + +def test_parse_env_cfg_preserves_task_device_when_omitted(): + """An omitted device should preserve a task-specific simulation requirement.""" + env_cfg = parse_env_cfg("IsaacContrib-Stack-Cube-Galbot-Right-Arm-Suction-RmpFlow") + + assert env_cfg.sim.device == "cpu" + env_cfg.validate() + + +def test_parse_env_cfg_applies_explicit_device_override(): + """An explicit device should continue to override the registered task default.""" + env_cfg = parse_env_cfg("Isaac-Cartpole", device="cpu") + + assert env_cfg.sim.device == "cpu" From 7127f50cca7c6f3cce04ad7cff397abe689b7c93 Mon Sep 17 00:00:00 2001 From: "isaaclab-bot[bot]" <282401363+isaaclab-bot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 05:12:02 +0000 Subject: [PATCH 017/128] [CI][Auto Version Bump] Compile changelog fragments (schedule) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumped packages: - isaaclab: 24.0.0 → 24.1.0 - isaaclab_newton: 6.1.1 → 6.2.0 - isaaclab_ov: 3.0.2 → 3.1.0 - isaaclab_physx: 7.0.1 → 7.1.0 - isaaclab_rl: 0.17.0 → 0.17.1 - isaaclab_tasks: 20.1.0 → 20.1.1 - isaaclab_visualizers: 1.10.2 → 1.10.3 --- ...antoiner-first-contact-tolerance.minor.rst | 13 --------- .../fix-benchmark-agent-library-wiring.rst | 7 ----- .../spacemouse-partial-init-shutdown.rst | 5 ---- source/isaaclab/docs/CHANGELOG.rst | 24 ++++++++++++++++ source/isaaclab/pyproject.toml | 2 +- ...antoiner-first-contact-tolerance.minor.rst | 12 -------- .../kguo-deformable-render-graph.rst | 4 --- .../neozng-mjwarp-usd-joint-properties.rst | 5 ---- source/isaaclab_newton/docs/CHANGELOG.rst | 20 +++++++++++++ source/isaaclab_newton/pyproject.toml | 2 +- ...antoiner-first-contact-tolerance.minor.rst | 12 -------- .../changelog.d/kitless-pxr-thread-limit.skip | 0 source/isaaclab_ov/docs/CHANGELOG.rst | 17 +++++++++++ source/isaaclab_ov/pyproject.toml | 2 +- ...antoiner-first-contact-tolerance.minor.rst | 12 -------- .../changelog.d/kguo-empty-rtx-frame.rst | 4 --- source/isaaclab_physx/docs/CHANGELOG.rst | 18 ++++++++++++ source/isaaclab_physx/pyproject.toml | 2 +- ...llyg-fix-galbot-surface-gripper-device.rst | 4 --- .../mustafa-pretrained-checkpoint-presets.rst | 6 ---- source/isaaclab_rl/docs/CHANGELOG.rst | 12 ++++++++ source/isaaclab_rl/pyproject.toml | 2 +- .../fix-franka-reach-diffik-teleop.rst | 5 ---- .../fix-preset-agent-auto-selection.rst | 8 ------ .../fix-skip-drlegs-walk-contrib-test.skip | 3 -- ...llyg-fix-galbot-surface-gripper-device.rst | 6 ---- ...mustafa-pretrained-checkpoint-presets.skip | 1 - .../surface-gripper-observation-shape.rst | 5 ---- .../unify-velocity-solver-inputs.rst | 7 ----- source/isaaclab_tasks/docs/CHANGELOG.rst | 28 +++++++++++++++++++ source/isaaclab_tasks/pyproject.toml | 2 +- .../fix-newton-visualizer-viewer-teardown.rst | 7 ----- .../isaaclab_visualizers/docs/CHANGELOG.rst | 12 ++++++++ source/isaaclab_visualizers/pyproject.toml | 2 +- 34 files changed, 138 insertions(+), 133 deletions(-) delete mode 100644 source/isaaclab/changelog.d/antoiner-first-contact-tolerance.minor.rst delete mode 100644 source/isaaclab/changelog.d/fix-benchmark-agent-library-wiring.rst delete mode 100644 source/isaaclab/changelog.d/spacemouse-partial-init-shutdown.rst delete mode 100644 source/isaaclab_newton/changelog.d/antoiner-first-contact-tolerance.minor.rst delete mode 100644 source/isaaclab_newton/changelog.d/kguo-deformable-render-graph.rst delete mode 100644 source/isaaclab_newton/changelog.d/neozng-mjwarp-usd-joint-properties.rst delete mode 100644 source/isaaclab_ov/changelog.d/antoiner-first-contact-tolerance.minor.rst delete mode 100644 source/isaaclab_ov/changelog.d/kitless-pxr-thread-limit.skip delete mode 100644 source/isaaclab_physx/changelog.d/antoiner-first-contact-tolerance.minor.rst delete mode 100644 source/isaaclab_physx/changelog.d/kguo-empty-rtx-frame.rst delete mode 100644 source/isaaclab_rl/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst delete mode 100644 source/isaaclab_rl/changelog.d/mustafa-pretrained-checkpoint-presets.rst delete mode 100644 source/isaaclab_tasks/changelog.d/fix-franka-reach-diffik-teleop.rst delete mode 100644 source/isaaclab_tasks/changelog.d/fix-preset-agent-auto-selection.rst delete mode 100644 source/isaaclab_tasks/changelog.d/fix-skip-drlegs-walk-contrib-test.skip delete mode 100644 source/isaaclab_tasks/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst delete mode 100644 source/isaaclab_tasks/changelog.d/mustafa-pretrained-checkpoint-presets.skip delete mode 100644 source/isaaclab_tasks/changelog.d/surface-gripper-observation-shape.rst delete mode 100644 source/isaaclab_tasks/changelog.d/unify-velocity-solver-inputs.rst delete mode 100644 source/isaaclab_visualizers/changelog.d/fix-newton-visualizer-viewer-teardown.rst diff --git a/source/isaaclab/changelog.d/antoiner-first-contact-tolerance.minor.rst b/source/isaaclab/changelog.d/antoiner-first-contact-tolerance.minor.rst deleted file mode 100644 index 910d7023a8c6..000000000000 --- a/source/isaaclab/changelog.d/antoiner-first-contact-tolerance.minor.rst +++ /dev/null @@ -1,13 +0,0 @@ -Fixed -^^^^^ - -* Fixed :meth:`~isaaclab.sensors.contact_sensor.BaseContactSensor.compute_first_contact` and - :meth:`~isaaclab.sensors.contact_sensor.BaseContactSensor.compute_first_air` silently missing - touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). Their - ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update interval - instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 rounding - error of the sensor clock, so most transitions were dropped. Callers that relied on the previous - behavior can pass ``abs_tol=1e-8`` explicitly. - Both methods now also refresh outdated sensor buffers before comparing, so a sensor with - ``history_length=0`` no longer reports the previous step's transitions when it is queried before - its data is read. diff --git a/source/isaaclab/changelog.d/fix-benchmark-agent-library-wiring.rst b/source/isaaclab/changelog.d/fix-benchmark-agent-library-wiring.rst deleted file mode 100644 index 0152650f6112..000000000000 --- a/source/isaaclab/changelog.d/fix-benchmark-agent-library-wiring.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fixed -^^^^^ - -* Fixed the ``rsl_rl``, ``rl_games`` and ``sb3`` benchmark train and play entrypoints not passing - ``agent_library`` to :func:`~isaaclab_tasks.utils.setup_preset_cli`, which disabled preset-based - ``--agent`` selection and the registered-agent help listing for those backends. Only the ``skrl`` - entrypoints wired it. diff --git a/source/isaaclab/changelog.d/spacemouse-partial-init-shutdown.rst b/source/isaaclab/changelog.d/spacemouse-partial-init-shutdown.rst deleted file mode 100644 index 8104eb2d74cb..000000000000 --- a/source/isaaclab/changelog.d/spacemouse-partial-init-shutdown.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fixed -^^^^^ - -* Fixed ``Se3SpaceMouse`` cleanup raising an exception when device initialization failed before - starting its listener thread. diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index c85482d5150b..406a4bb86527 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -1,6 +1,30 @@ Changelog --------- +24.1.0 (2026-09-08) +~~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed the ``rsl_rl``, ``rl_games`` and ``sb3`` benchmark train and play entrypoints not passing + ``agent_library`` to :func:`~isaaclab_tasks.utils.setup_preset_cli`, which disabled preset-based + ``--agent`` selection and the registered-agent help listing for those backends. Only the ``skrl`` + entrypoints wired it. +* Fixed :meth:`~isaaclab.sensors.contact_sensor.BaseContactSensor.compute_first_contact` and + :meth:`~isaaclab.sensors.contact_sensor.BaseContactSensor.compute_first_air` silently missing + touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). Their + ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update interval + instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 rounding + error of the sensor clock, so most transitions were dropped. Callers that relied on the previous + behavior can pass ``abs_tol=1e-8`` explicitly. + Both methods now also refresh outdated sensor buffers before comparing, so a sensor with + ``history_length=0`` no longer reports the previous step's transitions when it is queried before + its data is read. +* Fixed ``Se3SpaceMouse`` cleanup raising an exception when device initialization failed before + starting its listener thread. + + 24.0.0 (2026-09-07) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab/pyproject.toml b/source/isaaclab/pyproject.toml index 98ee35130675..9c0a91eb767d 100644 --- a/source/isaaclab/pyproject.toml +++ b/source/isaaclab/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "isaaclab" -version = "24.0.0" +version = "24.1.0" description = "Extension providing main framework interfaces and abstractions for robot learning." license = {text = "BSD-3-Clause"} authors = [{name = "Isaac Lab Project Developers"}] diff --git a/source/isaaclab_newton/changelog.d/antoiner-first-contact-tolerance.minor.rst b/source/isaaclab_newton/changelog.d/antoiner-first-contact-tolerance.minor.rst deleted file mode 100644 index f7d9db23c442..000000000000 --- a/source/isaaclab_newton/changelog.d/antoiner-first-contact-tolerance.minor.rst +++ /dev/null @@ -1,12 +0,0 @@ -Fixed -^^^^^ - -* Fixed :meth:`compute_first_contact` and :meth:`compute_first_air` on the contact sensor silently - missing touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). - Their ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update - interval instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 - rounding error of the sensor clock, so most transitions were dropped. Callers that relied on the - previous behavior can pass ``abs_tol=1e-8`` explicitly. - Both methods now also refresh outdated sensor buffers before comparing, so a sensor with - ``history_length=0`` no longer reports the previous step's transitions when it is queried before - its data is read. diff --git a/source/isaaclab_newton/changelog.d/kguo-deformable-render-graph.rst b/source/isaaclab_newton/changelog.d/kguo-deformable-render-graph.rst deleted file mode 100644 index 18e920b0e0df..000000000000 --- a/source/isaaclab_newton/changelog.d/kguo-deformable-render-graph.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixed -^^^^^ - -* Avoided conditional CUDA graph capture for Newton Warp rendering of deformable triangle meshes. diff --git a/source/isaaclab_newton/changelog.d/neozng-mjwarp-usd-joint-properties.rst b/source/isaaclab_newton/changelog.d/neozng-mjwarp-usd-joint-properties.rst deleted file mode 100644 index 87714973c404..000000000000 --- a/source/isaaclab_newton/changelog.d/neozng-mjwarp-usd-joint-properties.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fixed -^^^^^ - -* Fixed MuJoCo-based solver managers dropping ``mjc:frictionloss`` during USD - stage imports. diff --git a/source/isaaclab_newton/docs/CHANGELOG.rst b/source/isaaclab_newton/docs/CHANGELOG.rst index dd8c91b391c8..a71ff63d2373 100644 --- a/source/isaaclab_newton/docs/CHANGELOG.rst +++ b/source/isaaclab_newton/docs/CHANGELOG.rst @@ -1,6 +1,26 @@ Changelog --------- +6.2.0 (2026-09-08) +~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Avoided conditional CUDA graph capture for Newton Warp rendering of deformable triangle meshes. +* Fixed :meth:`compute_first_contact` and :meth:`compute_first_air` on the contact sensor silently + missing touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). + Their ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update + interval instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 + rounding error of the sensor clock, so most transitions were dropped. Callers that relied on the + previous behavior can pass ``abs_tol=1e-8`` explicitly. + Both methods now also refresh outdated sensor buffers before comparing, so a sensor with + ``history_length=0`` no longer reports the previous step's transitions when it is queried before + its data is read. +* Fixed MuJoCo-based solver managers dropping ``mjc:frictionloss`` during USD + stage imports. + + 6.1.1 (2026-09-07) ~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab_newton/pyproject.toml b/source/isaaclab_newton/pyproject.toml index 6f367bce18d4..016637308a88 100644 --- a/source/isaaclab_newton/pyproject.toml +++ b/source/isaaclab_newton/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "isaaclab_newton" -version = "6.1.1" +version = "6.2.0" description = "Extension providing IsaacLab with Newton specific abstractions." license = {text = "BSD-3-Clause"} authors = [{name = "Isaac Lab Project Developers"}] diff --git a/source/isaaclab_ov/changelog.d/antoiner-first-contact-tolerance.minor.rst b/source/isaaclab_ov/changelog.d/antoiner-first-contact-tolerance.minor.rst deleted file mode 100644 index f7d9db23c442..000000000000 --- a/source/isaaclab_ov/changelog.d/antoiner-first-contact-tolerance.minor.rst +++ /dev/null @@ -1,12 +0,0 @@ -Fixed -^^^^^ - -* Fixed :meth:`compute_first_contact` and :meth:`compute_first_air` on the contact sensor silently - missing touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). - Their ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update - interval instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 - rounding error of the sensor clock, so most transitions were dropped. Callers that relied on the - previous behavior can pass ``abs_tol=1e-8`` explicitly. - Both methods now also refresh outdated sensor buffers before comparing, so a sensor with - ``history_length=0`` no longer reports the previous step's transitions when it is queried before - its data is read. diff --git a/source/isaaclab_ov/changelog.d/kitless-pxr-thread-limit.skip b/source/isaaclab_ov/changelog.d/kitless-pxr-thread-limit.skip deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/source/isaaclab_ov/docs/CHANGELOG.rst b/source/isaaclab_ov/docs/CHANGELOG.rst index 14e3da95918d..ebc2eda7b2f1 100644 --- a/source/isaaclab_ov/docs/CHANGELOG.rst +++ b/source/isaaclab_ov/docs/CHANGELOG.rst @@ -1,6 +1,23 @@ Changelog --------- +3.1.0 (2026-09-08) +~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed :meth:`compute_first_contact` and :meth:`compute_first_air` on the contact sensor silently + missing touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). + Their ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update + interval instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 + rounding error of the sensor clock, so most transitions were dropped. Callers that relied on the + previous behavior can pass ``abs_tol=1e-8`` explicitly. + Both methods now also refresh outdated sensor buffers before comparing, so a sensor with + ``history_length=0`` no longer reports the previous step's transitions when it is queried before + its data is read. + + 3.0.2 (2026-09-07) ~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab_ov/pyproject.toml b/source/isaaclab_ov/pyproject.toml index d72da96f6c66..50fedca2b552 100644 --- a/source/isaaclab_ov/pyproject.toml +++ b/source/isaaclab_ov/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "isaaclab_ov" -version = "3.0.2" +version = "3.1.0" description = "Extension providing Omniverse rendering and OVPhysX simulation integrations." license = {text = "BSD-3-Clause"} authors = [{name = "Isaac Lab Project Developers"}] diff --git a/source/isaaclab_physx/changelog.d/antoiner-first-contact-tolerance.minor.rst b/source/isaaclab_physx/changelog.d/antoiner-first-contact-tolerance.minor.rst deleted file mode 100644 index f7d9db23c442..000000000000 --- a/source/isaaclab_physx/changelog.d/antoiner-first-contact-tolerance.minor.rst +++ /dev/null @@ -1,12 +0,0 @@ -Fixed -^^^^^ - -* Fixed :meth:`compute_first_contact` and :meth:`compute_first_air` on the contact sensor silently - missing touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). - Their ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update - interval instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 - rounding error of the sensor clock, so most transitions were dropped. Callers that relied on the - previous behavior can pass ``abs_tol=1e-8`` explicitly. - Both methods now also refresh outdated sensor buffers before comparing, so a sensor with - ``history_length=0`` no longer reports the previous step's transitions when it is queried before - its data is read. diff --git a/source/isaaclab_physx/changelog.d/kguo-empty-rtx-frame.rst b/source/isaaclab_physx/changelog.d/kguo-empty-rtx-frame.rst deleted file mode 100644 index ee44715caa3e..000000000000 --- a/source/isaaclab_physx/changelog.d/kguo-empty-rtx-frame.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixed -^^^^^ - -* Handled empty Isaac RTX annotator warm-up frames without invalid Warp slicing. diff --git a/source/isaaclab_physx/docs/CHANGELOG.rst b/source/isaaclab_physx/docs/CHANGELOG.rst index 23341ae4895d..b76b75e908a0 100644 --- a/source/isaaclab_physx/docs/CHANGELOG.rst +++ b/source/isaaclab_physx/docs/CHANGELOG.rst @@ -1,6 +1,24 @@ Changelog --------- +7.1.0 (2026-09-08) +~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Handled empty Isaac RTX annotator warm-up frames without invalid Warp slicing. +* Fixed :meth:`compute_first_contact` and :meth:`compute_first_air` on the contact sensor silently + missing touchdowns and lift-offs once the simulation had run for a few seconds (issue #7283). + Their ``abs_tol`` argument now defaults to ``None``, which resolves to half the sensor update + interval instead of a fixed ``1e-8``. The old value was around 100x smaller than the float32 + rounding error of the sensor clock, so most transitions were dropped. Callers that relied on the + previous behavior can pass ``abs_tol=1e-8`` explicitly. + Both methods now also refresh outdated sensor buffers before comparing, so a sensor with + ``history_length=0`` no longer reports the previous step's transitions when it is queried before + its data is read. + + 7.0.1 (2026-09-06) ~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab_physx/pyproject.toml b/source/isaaclab_physx/pyproject.toml index d7155d71cf08..36863f9ffa65 100644 --- a/source/isaaclab_physx/pyproject.toml +++ b/source/isaaclab_physx/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "isaaclab_physx" -version = "7.0.1" +version = "7.1.0" description = "Extension providing IsaacLab with PhysX specific abstractions." license = {text = "BSD-3-Clause"} authors = [{name = "Isaac Lab Project Developers"}] diff --git a/source/isaaclab_rl/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst b/source/isaaclab_rl/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst deleted file mode 100644 index 9fae38116873..000000000000 --- a/source/isaaclab_rl/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixed -^^^^^ - -* Fixed the zero and random agents overriding task-defined simulation devices when ``--device`` was omitted. diff --git a/source/isaaclab_rl/changelog.d/mustafa-pretrained-checkpoint-presets.rst b/source/isaaclab_rl/changelog.d/mustafa-pretrained-checkpoint-presets.rst deleted file mode 100644 index e9626aa27569..000000000000 --- a/source/isaaclab_rl/changelog.d/mustafa-pretrained-checkpoint-presets.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fixed -^^^^^ - -* Fixed published checkpoint lookup ignoring non-default domain presets, which could fetch an - incompatible policy or miss an available preset-specific checkpoint. Preset-specific checkpoints - can now also be trained, collected, reviewed, and published through the checkpoint management tool. diff --git a/source/isaaclab_rl/docs/CHANGELOG.rst b/source/isaaclab_rl/docs/CHANGELOG.rst index 6d4c4bce4c37..3ffe41167bc2 100644 --- a/source/isaaclab_rl/docs/CHANGELOG.rst +++ b/source/isaaclab_rl/docs/CHANGELOG.rst @@ -1,6 +1,18 @@ Changelog --------- +0.17.1 (2026-09-08) +~~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed published checkpoint lookup ignoring non-default domain presets, which could fetch an + incompatible policy or miss an available preset-specific checkpoint. Preset-specific checkpoints + can now also be trained, collected, reviewed, and published through the checkpoint management tool. +* Fixed the zero and random agents overriding task-defined simulation devices when ``--device`` was omitted. + + 0.17.0 (2026-09-05) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab_rl/pyproject.toml b/source/isaaclab_rl/pyproject.toml index 6333748542a0..07835963b15d 100644 --- a/source/isaaclab_rl/pyproject.toml +++ b/source/isaaclab_rl/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "isaaclab_rl" -version = "0.17.0" +version = "0.17.1" description = "Extension containing reinforcement learning related utilities." license = {text = "BSD-3-Clause"} authors = [{name = "Isaac Lab Project Developers"}] diff --git a/source/isaaclab_tasks/changelog.d/fix-franka-reach-diffik-teleop.rst b/source/isaaclab_tasks/changelog.d/fix-franka-reach-diffik-teleop.rst deleted file mode 100644 index 3a61cf601afc..000000000000 --- a/source/isaaclab_tasks/changelog.d/fix-franka-reach-diffik-teleop.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fixed -^^^^^ - -* Fixed native keyboard, gamepad, and SpaceMouse teleoperation for ``Isaac-Reach-Franka`` with the - ``diffik`` and ``newton_ik`` presets by disabling the unsupported gripper command. diff --git a/source/isaaclab_tasks/changelog.d/fix-preset-agent-auto-selection.rst b/source/isaaclab_tasks/changelog.d/fix-preset-agent-auto-selection.rst deleted file mode 100644 index 636eea3b20f8..000000000000 --- a/source/isaaclab_tasks/changelog.d/fix-preset-agent-auto-selection.rst +++ /dev/null @@ -1,8 +0,0 @@ -Fixed -^^^^^ - -* Fixed preset-based ``--agent`` auto-selection being skipped for every entrypoint that registers - ``--agent`` with a non-``None`` default (``rsl_rl``, ``rl_games`` and ``sb3``). The selection guard - could not tell a default-supplied value from a user-typed one, so ``presets=resnet18`` and - ``presets=theia_tiny`` on ``Isaac-Cartpole-Camera`` kept the raw-camera entry point and the runner - failed to construct. An explicitly typed ``--agent`` still wins over auto-selection. diff --git a/source/isaaclab_tasks/changelog.d/fix-skip-drlegs-walk-contrib-test.skip b/source/isaaclab_tasks/changelog.d/fix-skip-drlegs-walk-contrib-test.skip deleted file mode 100644 index e29cd024514a..000000000000 --- a/source/isaaclab_tasks/changelog.d/fix-skip-drlegs-walk-contrib-test.skip +++ /dev/null @@ -1,3 +0,0 @@ -Test-only change: skip ``IsaacContrib-DrLegs-Walk`` (and its deprecated alias) in the contributed -environment smoke test because the Kamino solver intermittently diverges to NaN under random actions. -No user-visible behavior change. diff --git a/source/isaaclab_tasks/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst b/source/isaaclab_tasks/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst deleted file mode 100644 index 2cf490f53e0c..000000000000 --- a/source/isaaclab_tasks/changelog.d/kellyg-fix-galbot-surface-gripper-device.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fixed -^^^^^ - -* Fixed surface-gripper stack tasks to select CPU simulation by default and reject unsupported GPU overrides before - simulator initialization. Task-defined simulation devices are now preserved by :func:`parse_env_cfg` when no - explicit device override is provided. diff --git a/source/isaaclab_tasks/changelog.d/mustafa-pretrained-checkpoint-presets.skip b/source/isaaclab_tasks/changelog.d/mustafa-pretrained-checkpoint-presets.skip deleted file mode 100644 index 129cb0981603..000000000000 --- a/source/isaaclab_tasks/changelog.d/mustafa-pretrained-checkpoint-presets.skip +++ /dev/null @@ -1 +0,0 @@ -Declared preset-specific checkpoint availability for the environment browser. diff --git a/source/isaaclab_tasks/changelog.d/surface-gripper-observation-shape.rst b/source/isaaclab_tasks/changelog.d/surface-gripper-observation-shape.rst deleted file mode 100644 index b394ee2c4c1c..000000000000 --- a/source/isaaclab_tasks/changelog.d/surface-gripper-observation-shape.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fixed -^^^^^ - -* Fixed surface-gripper stack and place observations returning a quadratic environment batch due to unintended - broadcasting. diff --git a/source/isaaclab_tasks/changelog.d/unify-velocity-solver-inputs.rst b/source/isaaclab_tasks/changelog.d/unify-velocity-solver-inputs.rst deleted file mode 100644 index c29d3af3d725..000000000000 --- a/source/isaaclab_tasks/changelog.d/unify-velocity-solver-inputs.rst +++ /dev/null @@ -1,7 +0,0 @@ -Changed -^^^^^^^ - -* Unified rough-velocity task inputs across physics backends by removing MJWarp-only actuator armatures, - using 5,000 G1 training iterations for every backend, and representing shared base-COM randomization as a - plain event. Downstream configurations that require the former backend-specific behavior should set it - explicitly. diff --git a/source/isaaclab_tasks/docs/CHANGELOG.rst b/source/isaaclab_tasks/docs/CHANGELOG.rst index 6e225a219503..8b8c9966c354 100644 --- a/source/isaaclab_tasks/docs/CHANGELOG.rst +++ b/source/isaaclab_tasks/docs/CHANGELOG.rst @@ -1,6 +1,34 @@ Changelog --------- +20.1.1 (2026-09-08) +~~~~~~~~~~~~~~~~~~~ + +Changed +^^^^^^^ + +* Unified rough-velocity task inputs across physics backends by removing MJWarp-only actuator armatures, + using 5,000 G1 training iterations for every backend, and representing shared base-COM randomization as a + plain event. Downstream configurations that require the former backend-specific behavior should set it + explicitly. + +Fixed +^^^^^ + +* Fixed preset-based ``--agent`` auto-selection being skipped for every entrypoint that registers + ``--agent`` with a non-``None`` default (``rsl_rl``, ``rl_games`` and ``sb3``). The selection guard + could not tell a default-supplied value from a user-typed one, so ``presets=resnet18`` and + ``presets=theia_tiny`` on ``Isaac-Cartpole-Camera`` kept the raw-camera entry point and the runner + failed to construct. An explicitly typed ``--agent`` still wins over auto-selection. +* Fixed surface-gripper stack and place observations returning a quadratic environment batch due to unintended + broadcasting. +* Fixed native keyboard, gamepad, and SpaceMouse teleoperation for ``Isaac-Reach-Franka`` with the + ``diffik`` and ``newton_ik`` presets by disabling the unsupported gripper command. +* Fixed surface-gripper stack tasks to select CPU simulation by default and reject unsupported GPU overrides before + simulator initialization. Task-defined simulation devices are now preserved by :func:`parse_env_cfg` when no + explicit device override is provided. + + 20.1.0 (2026-09-06) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab_tasks/pyproject.toml b/source/isaaclab_tasks/pyproject.toml index 7baacb75535f..53821fb4ae81 100644 --- a/source/isaaclab_tasks/pyproject.toml +++ b/source/isaaclab_tasks/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "isaaclab_tasks" -version = "20.1.0" +version = "20.1.1" description = "Extension containing suite of environments for robot learning." license = {text = "BSD-3-Clause"} authors = [{name = "Isaac Lab Project Developers"}] diff --git a/source/isaaclab_visualizers/changelog.d/fix-newton-visualizer-viewer-teardown.rst b/source/isaaclab_visualizers/changelog.d/fix-newton-visualizer-viewer-teardown.rst deleted file mode 100644 index 57f5b1edf599..000000000000 --- a/source/isaaclab_visualizers/changelog.d/fix-newton-visualizer-viewer-teardown.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fixed -^^^^^ - -* Fixed :class:`~isaaclab_visualizers.newton.newton_visualizer.NewtonRTXVisualizer` releasing its viewer - without first neutralizing picking callbacks and calling the viewer's :meth:`close`, which left its ordered - GPU teardown to the garbage collector and intermittently leaked render step results and attribute bindings - on shutdown. diff --git a/source/isaaclab_visualizers/docs/CHANGELOG.rst b/source/isaaclab_visualizers/docs/CHANGELOG.rst index 0b6c93d46436..b3f3457d0d19 100644 --- a/source/isaaclab_visualizers/docs/CHANGELOG.rst +++ b/source/isaaclab_visualizers/docs/CHANGELOG.rst @@ -1,6 +1,18 @@ Changelog --------- +1.10.3 (2026-09-08) +~~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed :class:`~isaaclab_visualizers.newton.newton_visualizer.NewtonRTXVisualizer` releasing its viewer + without first neutralizing picking callbacks and calling the viewer's :meth:`close`, which left its ordered + GPU teardown to the garbage collector and intermittently leaked render step results and attribute bindings + on shutdown. + + 1.10.2 (2026-09-04) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab_visualizers/pyproject.toml b/source/isaaclab_visualizers/pyproject.toml index 969cbed65af9..ac0b8740cadc 100644 --- a/source/isaaclab_visualizers/pyproject.toml +++ b/source/isaaclab_visualizers/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "isaaclab-visualizers" -version = "1.10.2" +version = "1.10.3" description = "Visualizer backends for Isaac Lab (Kit, Newton, Rerun, Viser)." license = {text = "BSD-3-Clause"} authors = [{name = "Isaac Lab Project Developers"}] From 25fad3eac7f5d9226dcf5f9361e40e9e867170dc Mon Sep 17 00:00:00 2001 From: Antoine RICHARD Date: Tue, 8 Sep 2026 16:28:19 +0200 Subject: [PATCH 018/128] Reduce direct locomotion step overhead (#6512) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Reduces common Direct Locomotion task overhead used by Ant and Humanoid without introducing the experimental Ant post-step path. The change: - stages effort targets once per environment step because articulation command buffers persist across decimation substeps; - uses the cached all-joint target path instead of repeatedly uploading joint indices; - keeps scalar zero assignments on-device; - removes a duplicate articulation reset already performed by the scene reset; - removes redundant clones after advanced indexing; and - uses the common fused joint-state writer provided by PhysX, Newton, and Isaac Lab OV. The isolated device scalar change improved the Ant host-return measurement from 3.433 ms to 3.231 ms. The remaining changes remove repeated launches, index transfers, copies, and reset writes; the larger fused post-step and device-mask work is deliberately excluded. This is one focused slice of superseded Draft PR #6509 and is independent of benchmark PR #6474. ## Type of change - Bug fix (non-breaking change which fixes an issue) ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable. ## Validation - `uv run python tools/changelog/cli.py check develop` - `uv run --frozen isaaclab -f` - `uv run --frozen --extra test python -m pytest source/isaaclab_tasks/test/core/test_environments_newton.py -k 'Direct and (Ant or Humanoid)' -q` (`2 passed`) - Verified the fused writer interface on PhysX, Newton, and Isaac Lab OV. - The source change was exercised by the original focused CPU/CUDA validation before the PR split; new test files are intentionally omitted from this slice. ## Checklist Docker and GPU tests run on demand. Push the commits you want tested, then comment `run-ci` on the pull request. - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [x] I have made corresponding changes to the documentation (not required for this internal performance fix) - [x] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package (do **not** edit `CHANGELOG.rst` or bump `extension.toml` — CI handles that) - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- .../changelog.d/ant-direct-reset-overhead.rst | 5 ++++ .../core/locomotion/locomotion_direct_env.py | 25 +++++++++---------- 2 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 source/isaaclab_tasks/changelog.d/ant-direct-reset-overhead.rst diff --git a/source/isaaclab_tasks/changelog.d/ant-direct-reset-overhead.rst b/source/isaaclab_tasks/changelog.d/ant-direct-reset-overhead.rst new file mode 100644 index 000000000000..c8b6afa4a80a --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/ant-direct-reset-overhead.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Reduced direct locomotion step and reset overhead by staging actions once per environment step, + avoiding duplicate articulation resets, redundant state copies, and separate joint-state writes. diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/locomotion/locomotion_direct_env.py b/source/isaaclab_tasks/isaaclab_tasks/core/locomotion/locomotion_direct_env.py index b6401d0058bb..c16798766293 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/locomotion/locomotion_direct_env.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/locomotion/locomotion_direct_env.py @@ -46,8 +46,6 @@ def __init__(self, cfg: DirectRLEnvCfg, render_mode: str | None = None, **kwargs self.joint_gears[joint_ids] = torch.tensor(gears, device=self.sim.device) # the energy and joint-limit penalties weigh each joint by its gear relative to the largest one self.gear_ratio_scaled = self.joint_gears / torch.max(self.joint_gears) - joint_dof_idx, _ = self.robot.find_joints(".*", as_proxy=True) - self._joint_dof_idx = joint_dof_idx.warp # resolve against the sensor's own body list: its ordering is backend-specific and does not # necessarily match the articulation's body ordering self._feet_body_idx, _ = self.joint_wrench.find_bodies(self.cfg.feet_body_names) @@ -83,11 +81,13 @@ def _setup_scene(self): def _pre_physics_step(self, actions: torch.Tensor) -> None: self.actions = actions.clone() - - def _apply_action(self) -> None: # the action is clamped before scaling: unbounded joint efforts drive the solver to NaN forces = self.action_scale * self.joint_gears * torch.clamp(self.actions, -1.0, 1.0) - self.robot.set_joint_effort_target_index(target=forces, joint_ids=self._joint_dof_idx) + self.robot.set_joint_effort_target_index(target=forces) + + def _apply_action(self) -> None: + # Joint effort targets persist in the articulation command buffer across decimation substeps. + pass def _compute_intermediate_values(self): self.torso_position = self.robot.data.root_pos_w.torch @@ -100,7 +100,7 @@ def _compute_intermediate_values(self): # planar vector from the torso to the walk target to_target = self.targets - self.torso_position - to_target[:, 2] = 0.0 + to_target[:, 2].zero_() # alignment of the torso with the world up axis and with the direction to the target self.up_proj = -self.robot.data.projected_gravity_b.torch[:, 2] @@ -188,27 +188,26 @@ def _reset_idx(self, env_ids: Sequence[int] | None): self.actions[env_ids] = 0.0 # root state is reset to the default pose, offset into the environment - default_root_pose = self.robot.data.default_root_pose.torch[env_ids].clone() + default_root_pose = self.robot.data.default_root_pose.torch[env_ids] default_root_pose[:, :3] += self.scene.env_origins[env_ids] self.robot.write_root_pose_to_sim_index(root_pose=default_root_pose, env_ids=env_ids) self.robot.write_root_velocity_to_sim_index( - root_velocity=self.robot.data.default_root_vel.torch[env_ids].clone(), env_ids=env_ids + root_velocity=self.robot.data.default_root_vel.torch[env_ids], env_ids=env_ids ) # joint state is randomized around the default pose and clamped back into the joint limits - joint_pos = self.robot.data.default_joint_pos.torch[env_ids].clone() - joint_vel = self.robot.data.default_joint_vel.torch[env_ids].clone() + joint_pos = self.robot.data.default_joint_pos.torch[env_ids] + joint_vel = self.robot.data.default_joint_vel.torch[env_ids] joint_pos += sample_uniform(*self.cfg.initial_joint_pos_range, joint_pos.shape, joint_pos.device) joint_vel += sample_uniform(*self.cfg.initial_joint_vel_range, joint_vel.shape, joint_vel.device) joint_pos_limits = self.robot.data.soft_joint_pos_limits.torch[env_ids] joint_pos = joint_pos.clamp_(joint_pos_limits[..., 0], joint_pos_limits[..., 1]) joint_vel_limits = self.robot.data.soft_joint_vel_limits.torch[env_ids] joint_vel = joint_vel.clamp_(-joint_vel_limits, joint_vel_limits) - self.robot.write_joint_position_to_sim_index(position=joint_pos, env_ids=env_ids) - self.robot.write_joint_velocity_to_sim_index(velocity=joint_vel, env_ids=env_ids) + self.robot.write_joint_state_to_sim_index(position=joint_pos, velocity=joint_vel, env_ids=env_ids) to_target = self.targets[env_ids] - default_root_pose[:, :3] - to_target[:, 2] = 0.0 + to_target[:, 2].zero_() self.potentials[env_ids] = -torch.linalg.norm(to_target, ord=2, dim=-1) / self.step_dt self._compute_intermediate_values() From b182de03da0fae9809ae360817d3951916b933d6 Mon Sep 17 00:00:00 2001 From: Frank Lai NV Date: Tue, 8 Sep 2026 11:59:03 -0700 Subject: [PATCH 019/128] improve leapp export (#7326) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Small additive updates for leapp. 1. Pins the leapp version to latest 0.6.1 which improves robustness for slicing operations. Previously this wasn't supported but now new features should automatically enable more types of operations. 2. Adds graph level expected frequency to leapp yaml. This change helps downstream deployment set default frequency based on training configs. 3. Adds the Isaac Lab env.yaml to the exported leapp bundle for convenience. Isaac sim 6.2 will add leapp controller features. the env.yaml is required to set things up. This will make porting the policy and its environment much more convenient. ## Type of change - New feature (non-breaking change which adds functionality) - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package (do **not** edit `CHANGELOG.rst` or bump `extension.toml` — CI handles that) - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- .../exporting_policies_with_leapp.rst | 2 +- .../leapp/export_utils.py | 20 ++++++++++++++++++- .../leapp/rl_games/export.py | 10 +++++++++- .../leapp/rsl_rl/export.py | 14 ++++++++++++- .../leapp/sb3/export.py | 10 +++++++++- .../leapp/skrl/export.py | 10 +++++++++- .../changelog.d/leapp-export-metadata.rst | 4 ++++ 7 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 source/isaaclab_rl/changelog.d/leapp-export-metadata.rst diff --git a/docs/source/policy_deployment/05_leapp/exporting_policies_with_leapp.rst b/docs/source/policy_deployment/05_leapp/exporting_policies_with_leapp.rst index ca864dadce68..1f2124fb2887 100644 --- a/docs/source/policy_deployment/05_leapp/exporting_policies_with_leapp.rst +++ b/docs/source/policy_deployment/05_leapp/exporting_policies_with_leapp.rst @@ -97,7 +97,7 @@ checkpoint (or at a custom path). The directory contains: - **Exported model files** — ``.onnx`` (default) or ``.pt`` depending on the chosen backend. - **Export metadata** — LEAPP records the semantic information and wiring needed by downstream - deployment runtimes. + deployment runtimes, including the policy execution frequency. - **Initial values** — a ``.safetensors`` file for any feedback state, such as recurrent hidden state or last action. - **A graph visualization** — a ``.png`` diagram of the pipeline (can be disabled). diff --git a/scripts/reinforcement_learning/leapp/export_utils.py b/scripts/reinforcement_learning/leapp/export_utils.py index c2c69ec9a315..a4518c5171c5 100644 --- a/scripts/reinforcement_learning/leapp/export_utils.py +++ b/scripts/reinforcement_learning/leapp/export_utils.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Shared CLI and recurrent-state helpers for LEAPP policy export.""" +"""Shared CLI, graph metadata, and recurrent-state helpers for LEAPP policy export.""" from __future__ import annotations @@ -13,6 +13,9 @@ if TYPE_CHECKING: import torch + from leapp import GraphConfigs + + from isaaclab.envs import DirectRLEnvCfg, ManagerBasedEnvCfg def add_common_export_args(parser: argparse.ArgumentParser, *, agent_default: str) -> None: @@ -108,6 +111,21 @@ def disable_torchscript_for_export() -> None: torch.jit._state.disable() +def create_graph_configs(env_cfg: ManagerBasedEnvCfg | DirectRLEnvCfg) -> GraphConfigs: + """Create LEAPP graph metadata from an Isaac Lab environment configuration. + + Args: + env_cfg: Environment configuration that defines the policy period. + + Returns: + Graph metadata containing the policy frequency [Hz]. + """ + from leapp import GraphConfigs + + policy_frequency = 1.0 / (env_cfg.sim.dt * env_cfg.decimation) + return GraphConfigs(frequency=policy_frequency) + + def is_two_tensor_lstm_state(states: object) -> bool: """Return whether *states* looks like an LSTM ``[hidden, cell]`` state.""" import torch diff --git a/scripts/reinforcement_learning/leapp/rl_games/export.py b/scripts/reinforcement_learning/leapp/rl_games/export.py index b75dcabfe80d..03588b211733 100644 --- a/scripts/reinforcement_learning/leapp/rl_games/export.py +++ b/scripts/reinforcement_learning/leapp/rl_games/export.py @@ -40,6 +40,7 @@ is_two_tensor_lstm_state = None state_dict_from_sequence = None state_sequence_from_registered = None +create_graph_configs = None def parse_export_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[str]]: @@ -68,6 +69,7 @@ def _load_runtime_dependencies() -> None: global hydra_task_config, multi_agent_to_single_agent global patch_env_for_export, retrieve_file_path, torch, vecenv global is_two_tensor_lstm_state, state_dict_from_sequence, state_sequence_from_registered + global create_graph_configs if _RUNTIME_IMPORTS_LOADED: return @@ -97,6 +99,7 @@ def _load_runtime_dependencies() -> None: if _leapp_scripts_dir not in sys.path: sys.path.insert(0, _leapp_scripts_dir) from export_utils import ( # isort: skip + create_graph_configs as create_graph_configs_fn, is_two_tensor_lstm_state as is_two_tensor_lstm_state_fn, state_dict_from_sequence as state_dict_from_sequence_fn, state_sequence_from_registered as state_sequence_from_registered_fn, @@ -139,6 +142,7 @@ def _load_runtime_dependencies() -> None: is_two_tensor_lstm_state = is_two_tensor_lstm_state_fn state_dict_from_sequence = state_dict_from_sequence_fn state_sequence_from_registered = state_sequence_from_registered_fn + create_graph_configs = create_graph_configs_fn _RUNTIME_IMPORTS_LOADED = True @@ -306,7 +310,11 @@ def export_rl_games_agent( leapp.stop() leapp_started = False validate = args_cli.validation_steps > 0 - leapp.compile_graph(visualize=not args_cli.disable_graph_visualization, validate=validate) + leapp.compile_graph( + visualize=not args_cli.disable_graph_visualization, + validate=validate, + graph_configs=create_graph_configs(env_cfg), + ) finally: if leapp_started: with contextlib.suppress(Exception): diff --git a/scripts/reinforcement_learning/leapp/rsl_rl/export.py b/scripts/reinforcement_learning/leapp/rsl_rl/export.py index 34123c8e5a3e..71f1b06f05d6 100644 --- a/scripts/reinforcement_learning/leapp/rsl_rl/export.py +++ b/scripts/reinforcement_learning/leapp/rsl_rl/export.py @@ -41,6 +41,7 @@ get_checkpoint_path = None hydra_task_config = None installed_version = None +create_graph_configs = None def parse_export_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[str]]: @@ -71,6 +72,7 @@ def _load_runtime_dependencies() -> None: global handle_deprecated_rsl_rl_cfg, hydra_task_config global installed_version global patch_env_for_export, retrieve_file_path + global create_graph_configs if _RUNTIME_IMPORTS_LOADED: return @@ -105,6 +107,11 @@ def _load_runtime_dependencies() -> None: from isaaclab_tasks.utils import get_checkpoint_path as get_checkpoint_path_fn from isaaclab_tasks.utils.hydra import hydra_task_config as hydra_task_config_fn + _leapp_scripts_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + if _leapp_scripts_dir not in sys.path: + sys.path.insert(0, _leapp_scripts_dir) + from export_utils import create_graph_configs as create_graph_configs_fn + installed_version = metadata.version("rsl-rl-lib") if packaging_version_module.parse(installed_version) < packaging_version_module.parse(RSL_RL_MIN_VERSION): print( @@ -128,6 +135,7 @@ def _load_runtime_dependencies() -> None: get_published_pretrained_checkpoint = get_published_pretrained_checkpoint_fn get_checkpoint_path = get_checkpoint_path_fn hydra_task_config = hydra_task_config_fn + create_graph_configs = create_graph_configs_fn _RUNTIME_IMPORTS_LOADED = True @@ -345,7 +353,11 @@ def export_rsl_rl_agent( leapp.stop() leapp_started = False validate = args_cli.validation_steps > 0 - leapp.compile_graph(visualize=not args_cli.disable_graph_visualization, validate=validate) + leapp.compile_graph( + visualize=not args_cli.disable_graph_visualization, + validate=validate, + graph_configs=create_graph_configs(env_cfg), + ) finally: if leapp_started: with contextlib.suppress(Exception): diff --git a/scripts/reinforcement_learning/leapp/sb3/export.py b/scripts/reinforcement_learning/leapp/sb3/export.py index 3ce76171e993..9bfd8a24c87f 100644 --- a/scripts/reinforcement_learning/leapp/sb3/export.py +++ b/scripts/reinforcement_learning/leapp/sb3/export.py @@ -35,6 +35,7 @@ CHECKPOINT_SELECTORS = None state_dict_from_sequence = None state_sequence_from_registered = None +create_graph_configs = None def parse_export_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[str]]: @@ -57,6 +58,7 @@ def _load_runtime_dependencies() -> None: global get_published_pretrained_checkpoint global load_from_pkl, load_from_zip_file, patch_env_for_export, resolve_checkpoint_selector, retrieve_file_path global state_dict_from_sequence, state_sequence_from_registered, torch + global create_graph_configs if _RUNTIME_IMPORTS_LOADED: return @@ -103,6 +105,7 @@ def _load_runtime_dependencies() -> None: if leapp_scripts_dir not in sys.path: sys.path.insert(0, leapp_scripts_dir) from export_utils import ( # isort: skip + create_graph_configs as create_graph_configs_fn, state_dict_from_sequence as state_dict_from_sequence_fn, state_sequence_from_registered as state_sequence_from_registered_fn, ) @@ -126,6 +129,7 @@ def _load_runtime_dependencies() -> None: CHECKPOINT_SELECTORS = CHECKPOINT_SELECTORS_VALUE state_dict_from_sequence = state_dict_from_sequence_fn state_sequence_from_registered = state_sequence_from_registered_fn + create_graph_configs = create_graph_configs_fn _RUNTIME_IMPORTS_LOADED = True @@ -363,7 +367,11 @@ def export_sb3_agent( leapp.stop() leapp_started = False validate = args_cli.validation_steps > 0 - leapp.compile_graph(visualize=not args_cli.disable_graph_visualization, validate=validate) + leapp.compile_graph( + visualize=not args_cli.disable_graph_visualization, + validate=validate, + graph_configs=create_graph_configs(env_cfg), + ) finally: torch.distributions.Distribution.set_default_validate_args(previous_validate_args) if leapp_started: diff --git a/scripts/reinforcement_learning/leapp/skrl/export.py b/scripts/reinforcement_learning/leapp/skrl/export.py index 8b0f632e3d4d..d76574e73abc 100644 --- a/scripts/reinforcement_learning/leapp/skrl/export.py +++ b/scripts/reinforcement_learning/leapp/skrl/export.py @@ -38,6 +38,7 @@ is_two_tensor_lstm_state = None state_dict_from_sequence = None state_sequence_from_registered = None +create_graph_configs = None def parse_export_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[str]]: @@ -76,6 +77,7 @@ def _load_runtime_dependencies() -> None: global hydra_task_config, multi_agent_to_single_agent global patch_env_for_export, retrieve_file_path, skrl, torch, version global configure_seed, is_two_tensor_lstm_state, state_dict_from_sequence, state_sequence_from_registered + global create_graph_configs if _RUNTIME_IMPORTS_LOADED: return @@ -104,6 +106,7 @@ def _load_runtime_dependencies() -> None: if _leapp_scripts_dir not in sys.path: sys.path.insert(0, _leapp_scripts_dir) from export_utils import ( # isort: skip + create_graph_configs as create_graph_configs_fn, is_two_tensor_lstm_state as is_two_tensor_lstm_state_fn, state_dict_from_sequence as state_dict_from_sequence_fn, state_sequence_from_registered as state_sequence_from_registered_fn, @@ -150,6 +153,7 @@ def _load_runtime_dependencies() -> None: is_two_tensor_lstm_state = is_two_tensor_lstm_state_fn state_dict_from_sequence = state_dict_from_sequence_fn state_sequence_from_registered = state_sequence_from_registered_fn + create_graph_configs = create_graph_configs_fn _RUNTIME_IMPORTS_LOADED = True @@ -298,7 +302,11 @@ def export_skrl_agent( leapp.stop() leapp_started = False validate = args_cli.validation_steps > 0 - leapp.compile_graph(visualize=not args_cli.disable_graph_visualization, validate=validate) + leapp.compile_graph( + visualize=not args_cli.disable_graph_visualization, + validate=validate, + graph_configs=create_graph_configs(env_cfg), + ) finally: if leapp_started: with contextlib.suppress(Exception): diff --git a/source/isaaclab_rl/changelog.d/leapp-export-metadata.rst b/source/isaaclab_rl/changelog.d/leapp-export-metadata.rst new file mode 100644 index 000000000000..597b3a13ee4c --- /dev/null +++ b/source/isaaclab_rl/changelog.d/leapp-export-metadata.rst @@ -0,0 +1,4 @@ +Added +^^^^^ + +* Added policy frequency metadata to LEAPP export artifacts for all supported RL libraries. From c91b03687f4f4c753553b6d3617a75d988b973e5 Mon Sep 17 00:00:00 2001 From: Mustafa H <34825877+StafaH@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:03:41 -0400 Subject: [PATCH 020/128] [Workflow] Update extra paths configuration for all workflows (#6644) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Fixes #6475. This change makes Python import-path configuration consistent across Isaac Lab source checkouts, installed packages, and generated external projects, including both VS Code/Pylance and Cursor/basedpyright. The configuration is split by ownership: - Checked-in Pyright policy lives in `pyproject.toml`. - Machine-specific paths are written to a git-ignored `pyrightconfig.json` that extends the project policy. - VS Code settings retain interpreter and editor behavior without defining a competing `python.analysis.extraPaths` value. The shared `isaaclab.utils.editor` utility discovers: - Isaac Sim extension paths from the selected installation. - Isaac Lab monorepo packages under `source/*`. - Standard downstream packages under a `src` layout. - Isaac Lab packages visible to the active interpreter, covering editable and wheel installs. The repository setup script and installed-package workflow now use that shared utility. The template generator copies the canonical setup wrapper instead of maintaining a second implementation, and the generated `pyproject.toml`, README, tests, and documentation are aligned with the new single-package uv `src` layout introduced on `develop`. Invalid explicit `--isaac_path` values now fail clearly rather than silently selecting another installation. No new runtime dependency is required. ## Type of change - Bug fix (non-breaking change which fixes an issue) - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable. ## Validation - `uv run pytest source/isaaclab_rl/test/test_template_generator.py source/isaaclab/test/cli/test_installed_workflow_entrypoints.py -q` — 34 passed - `uv run pytest --confcutdir=tools/template tools/template/test_cli.py -q` — 7 passed - `uv run pytest source/isaaclab/test/cli/test_wheel_builder_metadata.py -q` — 11 passed - Pyright 1.1.411 configuration check — 0 errors and 0 warnings - `uv run isaaclab -f` — passed, including changelog validation against the current upstream `develop` - `uv run --isolated --extra dev -- make -C docs current-docs` — warning-free build passed ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the pre-commit checks with `uv run isaaclab -f` - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package - [x] My name already exists in `CONTRIBUTORS.md` --- .gitignore | 1 + .vscode/.gitignore | 1 - .vscode/tasks.json | 4 +- .vscode/tools/settings.template.json | 5 +- .vscode/tools/setup_vscode.py | 200 --------------- docs/index.rst | 1 + docs/source/developer-tools/editor_setup.rst | 203 +++++++++++++++ .../developer-tools/template_generator.rst | 20 +- .../source/overview/developer-guide/index.rst | 7 +- .../overview/developer-guide/vs_code.rst | 121 --------- docs/source/setup/installation/index.rst | 9 +- docs/source/setup/quickstart.rst | 20 ++ pyproject.toml | 5 + .../mhaiderbhai-fix-pyright-editor-setup.rst | 6 + source/isaaclab/isaaclab/__main__.py | 147 +---------- source/isaaclab/isaaclab/cli/__init__.py | 13 +- .../isaaclab/isaaclab/cli/commands/install.py | 6 +- source/isaaclab/isaaclab/cli/commands/misc.py | 32 ++- source/isaaclab/isaaclab/utils/editor.py | 235 ++++++++++++++++++ .../test/cli/test_install_command_parsing.py | 2 +- .../test_installed_workflow_entrypoints.py | 32 ++- .../mhaiderbhai-vscode-pyright-tests.skip | 1 + .../test/test_template_generator.py | 76 ++++++ tools/template/templates/external/.gitignore | 1 + .../templates/external/.vscode/.gitignore | 1 - .../templates/external/.vscode/tasks.json | 2 +- .../.vscode/tools/settings.template.json | 5 +- .../external/.vscode/tools/setup_vscode.py | 198 --------------- tools/template/templates/external/README.md | 35 ++- .../templates/external/pyproject.toml | 5 + 30 files changed, 670 insertions(+), 724 deletions(-) delete mode 100644 .vscode/tools/setup_vscode.py create mode 100644 docs/source/developer-tools/editor_setup.rst delete mode 100644 docs/source/overview/developer-guide/vs_code.rst create mode 100644 source/isaaclab/changelog.d/mhaiderbhai-fix-pyright-editor-setup.rst create mode 100644 source/isaaclab/isaaclab/utils/editor.py create mode 100644 source/isaaclab_rl/changelog.d/mhaiderbhai-vscode-pyright-tests.skip delete mode 100644 tools/template/templates/external/.vscode/tools/setup_vscode.py diff --git a/.gitignore b/.gitignore index bc429f201b3e..68ec0e77b294 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ _build # No benchmarks output /benchmarks/ benchmark_*.json +/pyrightconfig.json # Ruff cache diff --git a/.vscode/.gitignore b/.vscode/.gitignore index 10b0af342ce3..e0a8bea56c6f 100644 --- a/.vscode/.gitignore +++ b/.vscode/.gitignore @@ -1,7 +1,6 @@ # Note: These files are kept for development purposes only. !tools/launch.template.json !tools/settings.template.json -!tools/setup_vscode.py !extensions.json !tasks.json diff --git a/.vscode/tasks.json b/.vscode/tasks.json index f7896a038b45..19da23a46ed0 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -8,10 +8,10 @@ "label": "setup_python_env", "type": "shell", "linux": { - "command": "${workspaceFolder}/isaaclab.sh -p ${workspaceFolder}/.vscode/tools/setup_vscode.py" + "command": "uv run isaaclab --editor" }, "windows": { - "command": "${workspaceFolder}/isaaclab.bat -p ${workspaceFolder}/.vscode/tools/setup_vscode.py" + "command": "uv run isaaclab --editor" } }, { diff --git a/.vscode/tools/settings.template.json b/.vscode/tools/settings.template.json index 4b07a6a8f9ad..e9ade2972128 100644 --- a/.vscode/tools/settings.template.json +++ b/.vscode/tools/settings.template.json @@ -78,8 +78,5 @@ }, "[restructuredtext]": { "editor.tabSize": 2 - }, - // Python extra paths - // Note: this is filled up when "./isaaclab.sh -i" is run - "python.analysis.extraPaths": [] + } } diff --git a/.vscode/tools/setup_vscode.py b/.vscode/tools/setup_vscode.py deleted file mode 100644 index 8d29dafee080..000000000000 --- a/.vscode/tools/setup_vscode.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""This script sets up the vs-code settings for the Isaac Lab project. - -This script merges the python.analysis.extraPaths from the "{ISAACSIM_DIR}/.vscode/settings.json" file into -the ".vscode/settings.json" file. - -This is necessary because Isaac Sim 2022.2.1 onwards does not add the necessary python packages to the python path -when the "setup_python_env.sh" is run as part of the vs-code launch configuration. -""" - -import re -import subprocess -import sys -import os -import pathlib - - -ISAACLAB_DIR = pathlib.Path(__file__).parents[2] -"""Path to the Isaac Lab directory.""" - -# Try to find IsaacSim dir -_isaacsim_probe = subprocess.run( - [sys.executable, "-c", "import isaacsim; import os; print(os.environ.get('ISAAC_PATH', ''))"], - capture_output=True, - text=True, - check=False, - # avoid EULA prompt - stdin=subprocess.DEVNULL, -) -if _isaacsim_probe.returncode == 0 and _isaacsim_probe.stdout.strip(): - isaacsim_dir = _isaacsim_probe.stdout.strip() -else: - isaacsim_dir = os.path.join(ISAACLAB_DIR, "_isaac_sim") - -# check if the isaac-sim directory exists -if not os.path.exists(isaacsim_dir): - print( - f"[WARN] Could not find the isaac-sim directory: {isaacsim_dir}." - "\n\tIsaac Sim does not appear to be installed. VS Code settings will be generated" - "\n\twithout Isaac Sim extra paths." - ) - isaacsim_dir = "" - -ISAACSIM_DIR = isaacsim_dir -"""Path to the isaac-sim directory.""" - - -def overwrite_python_analysis_extra_paths(isaaclab_settings: str) -> str: - """Overwrite the python.analysis.extraPaths in the Isaac Lab settings file. - - The extraPaths are replaced with the path names from the isaac-sim settings file that exists in the - "{ISAACSIM_DIR}/.vscode/settings.json" file. - - If the isaac-sim settings file does not exist, the extraPaths are not overwritten. - - Args: - isaaclab_settings: The settings string to use as template. - - Returns: - The settings string with overwritten python analysis extra paths. - """ - # isaac-sim settings - isaacsim_vscode_filename = os.path.join(ISAACSIM_DIR, ".vscode", "settings.json") - - # we use the isaac-sim settings file to get the python.analysis.extraPaths for kit extensions - # if this file does not exist, we will not add any extra paths - if ISAACSIM_DIR and os.path.exists(isaacsim_vscode_filename): - # read the path names from the isaac-sim settings file - with open(isaacsim_vscode_filename) as f: - vscode_settings = f.read() - # extract the path names - # search for the python.analysis.extraPaths section and extract the contents - settings = re.search( - r"\"python.analysis.extraPaths\": \[.*?\]", vscode_settings, flags=re.MULTILINE | re.DOTALL - ) - settings = settings.group(0) - settings = settings.split('"python.analysis.extraPaths": [')[-1] - settings = settings.split("]")[0] - - # read the path names from the isaac-sim settings file - path_names = settings.split(",") - path_names = [path_name.strip().strip('"') for path_name in path_names] - path_names = [path_name for path_name in path_names if len(path_name) > 0] - - # change the path names to be relative to the Isaac Lab directory - rel_path = os.path.relpath(ISAACSIM_DIR, ISAACLAB_DIR) - path_names = ['"${workspaceFolder}/' + rel_path + "/" + path_name + '"' for path_name in path_names] - else: - path_names = [] - - # add the path names that are in the Isaac Lab extensions directory - isaaclab_extensions = os.listdir(os.path.join(ISAACLAB_DIR, "source")) - path_names.extend(['"${workspaceFolder}/source/' + ext + '"' for ext in isaaclab_extensions]) - - # combine them into a single string - path_names = ",\n\t\t".expandtabs(4).join(path_names) - # deal with the path separator being different on Windows and Unix - path_names = path_names.replace("\\", "/") - - # replace the path names in the Isaac Lab settings file with the path names parsed - isaaclab_settings = re.sub( - r"\"python.analysis.extraPaths\": \[.*?\]", - '"python.analysis.extraPaths": [\n\t\t'.expandtabs(4) + path_names + "\n\t]".expandtabs(4), - isaaclab_settings, - flags=re.DOTALL, - ) - # return the Isaac Lab settings string - return isaaclab_settings - - -def overwrite_default_python_interpreter(isaaclab_settings: str) -> str: - """Overwrite the default python interpreter in the Isaac Lab settings file. - - The default python interpreter is replaced with the path to the python interpreter used by the - isaac-sim project. This is necessary because the default python interpreter is the one shipped with - isaac-sim. - - Args: - isaaclab_settings: The settings string to use as template. - - Returns: - The settings string with overwritten default python interpreter. - """ - # read executable name - python_exe = sys.executable.replace("\\", "/") - - # We make an exception for replacing the default interpreter if the - # path (/kit/python/bin/python3) indicates that we are using a local/container - # installation of IsaacSim. We will preserve the calling script as the default, python.sh. - # We want to use python.sh because it modifies LD_LIBRARY_PATH and PYTHONPATH - # (among other envars) that we need for all of our dependencies to be accessible. - if "kit/python/bin/python3" in python_exe: - return isaaclab_settings - # replace the default python interpreter in the Isaac Lab settings file with the path to the - # python interpreter in the Isaac Lab directory - isaaclab_settings = re.sub( - r"\"python.defaultInterpreterPath\": \".*?\"", - f'"python.defaultInterpreterPath": "{python_exe}"', - isaaclab_settings, - flags=re.DOTALL, - ) - # return the Isaac Lab settings file - return isaaclab_settings - - -def main(): - # Isaac Lab template settings - isaaclab_vscode_template_filename = os.path.join(ISAACLAB_DIR, ".vscode", "tools", "settings.template.json") - # make sure the Isaac Lab template settings file exists - if not os.path.exists(isaaclab_vscode_template_filename): - raise FileNotFoundError( - f"Could not find the Isaac Lab template settings file: {isaaclab_vscode_template_filename}" - ) - # read the Isaac Lab template settings file - with open(isaaclab_vscode_template_filename) as f: - isaaclab_template_settings = f.read() - - # overwrite the python.analysis.extraPaths in the Isaac Lab settings file with the path names - isaaclab_settings = overwrite_python_analysis_extra_paths(isaaclab_template_settings) - # overwrite the default python interpreter in the Isaac Lab settings file with the path to the - # python interpreter used to call this script - isaaclab_settings = overwrite_default_python_interpreter(isaaclab_settings) - - # add template notice to the top of the file - header_message = ( - "// This file is a template and is automatically generated by the setup_vscode.py script.\n" - "// Do not edit this file directly.\n" - "// \n" - f"// Generated from: {isaaclab_vscode_template_filename}\n" - ) - isaaclab_settings = header_message + isaaclab_settings - - # write the Isaac Lab settings file - isaaclab_vscode_filename = os.path.join(ISAACLAB_DIR, ".vscode", "settings.json") - with open(isaaclab_vscode_filename, "w") as f: - f.write(isaaclab_settings) - - # copy the launch.json file if it doesn't exist - isaaclab_vscode_launch_filename = os.path.join(ISAACLAB_DIR, ".vscode", "launch.json") - isaaclab_vscode_template_launch_filename = os.path.join(ISAACLAB_DIR, ".vscode", "tools", "launch.template.json") - if not os.path.exists(isaaclab_vscode_launch_filename): - # read template launch settings - with open(isaaclab_vscode_template_launch_filename) as f: - isaaclab_template_launch_settings = f.read() - # add header - header_message = header_message.replace( - isaaclab_vscode_template_filename, isaaclab_vscode_template_launch_filename - ) - isaaclab_launch_settings = header_message + isaaclab_template_launch_settings - # write the Isaac Lab launch settings file - with open(isaaclab_vscode_launch_filename, "w") as f: - f.write(isaaclab_launch_settings) - - -if __name__ == "__main__": - main() diff --git a/docs/index.rst b/docs/index.rst index 219c9fc9a3ea..3856bf9946fd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -149,6 +149,7 @@ Table of Contents :maxdepth: 2 :caption: Developer Tools + source/developer-tools/editor_setup source/developer-tools/template_generator source/developer-tools/benchmarking/index diff --git a/docs/source/developer-tools/editor_setup.rst b/docs/source/developer-tools/editor_setup.rst new file mode 100644 index 000000000000..05d54a151342 --- /dev/null +++ b/docs/source/developer-tools/editor_setup.rst @@ -0,0 +1,203 @@ +.. _setup-vs-code: + +Editor setup +------------ + +Editor setup is optional and is not required to run Isaac Lab. The repository +includes shared settings for `Visual Studio Code `_ +and compatible editors such as `Cursor `_. Complete one +of the :ref:`Isaac Lab installation methods ` before +configuring your editor. + +The ``.vscode`` directory contains the checked-in templates and tasks: + +.. code-block:: bash + + .vscode + ├── tools + │   ├── launch.template.json + │   └── settings.template.json + ├── extensions.json + ├── launch.json # generated by isaaclab --editor + ├── settings.json # generated by isaaclab --editor + └── tasks.json + +Configure a source checkout +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Open the Isaac Lab repository root in your editor. Then run the setup command +that matches your installation from a terminal in that directory. + +.. tab-set:: + + .. tab-item:: uv (recommended) + + For the default Newton environment, run: + + .. code-block:: bash + + uv run isaaclab --editor + + If you use Isaac Sim from the ``isaacsim`` extra, include the extra so the + setup command can discover its extensions: + + .. code-block:: bash + + uv run --extra isaacsim isaaclab --editor + + .. tab-item:: Activated Python environment + + Activate the uv, venv, or conda environment where Isaac Lab is installed, + then run: + + .. code-block:: bash + + isaaclab --editor + + .. tab-item:: Downloaded Isaac Sim package + + Run the setup command through the Isaac Lab launcher after completing the + :ref:`downloaded package installation `: + + .. tab-set:: + :sync-group: os + + .. tab-item:: :icon:`fa-brands fa-linux` Linux + :sync: linux + + .. code-block:: bash + + ./isaaclab.sh --editor + + .. tab-item:: :icon:`fa-brands fa-windows` Windows + :sync: windows + + .. code-block:: batch + + isaaclab.bat --editor + + The ``setup_python_env`` task in the command palette runs the recommended + ``uv`` workflow. + +The command creates or updates these machine-local files: + +* ``.vscode/launch.json``: Debugging configurations. An existing file is preserved. +* ``.vscode/settings.json``: The interpreter and shared editor settings. +* ``pyrightconfig.json``: Import paths for Pyright-compatible language servers. + +The generated files are ignored by Git because interpreter and extension paths +vary between machines. Rerun the command after changing Python environments or +Isaac Sim installations. If Isaac Sim is not installed, the command prints a +warning and still configures the local Isaac Lab packages. + +The checked-in ``[tool.pyright]`` table in ``pyproject.toml`` makes packages +under ``source`` available immediately after cloning. The generated +``pyrightconfig.json`` inherits that policy and adds Isaac Sim extensions plus +Isaac Lab packages found in the active Python environment. This covers source, +editable, and wheel installations without storing absolute paths in Git. + +Configure VS Code +^^^^^^^^^^^^^^^^^ + +Install the extensions recommended by the repository when VS Code prompts you. +At minimum, install the Python and Pylance extensions. Run the setup command +above, then use **Python: Select Interpreter** from the command palette to select +the same interpreter used by the command. For the recommended uv installation, +this is ``.venv/bin/python`` on Linux or ``.venv\Scripts\python.exe`` on Windows. + +Configure Cursor +^^^^^^^^^^^^^^^^ + +Cursor cannot use Pylance because Pylance is licensed for official VS Code +builds. Install the Python extension and the `basedpyright +`__ +extension (``detachhead.basedpyright``). Then: + +1. Run the same setup command shown above for your installation. +2. Select the interpreter that ran the command. +3. Reload the Cursor window so basedpyright rereads ``pyrightconfig.json``. + +No Cursor-specific path list is required. Pylance and basedpyright read the +same Pyright configuration. + +Troubleshoot editor imports +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If an import is still unresolved: + +1. Confirm the selected editor interpreter matches ``python`` in the setup command. +2. Rerun setup with ``uv run --extra isaacsim isaaclab --editor`` if the missing import is from + ``omni``, ``pxr``, or ``isaacsim``. +3. Reload the editor window. +4. Inspect the generated ``extraPaths`` in the root ``pyrightconfig.json``. + +Remove simulator extension directories that the project does not use if +language-server indexing consumes too much memory. + +For more information about VS Code support in Isaac Sim, see: + +* `Isaac Sim VSCode support `__ + + +Attach to a Running ``debugpy`` Session +--------------------------------------- + +The generated ``.vscode/launch.json`` includes a ``Python: Debugger Attach`` +configuration that starts to listen on port ``localhost:3000`` for the debugpy session. + +To use it: + +1. Set your breakpoints. +2. Run your code under debugpy like so: + + .. tab-set:: + + .. tab-item:: uv (Recommended) + + .. code-block:: bash + + uv run python -m debugpy --listen 3000 --wait-for-client -c "from isaaclab.cli import cli; cli()" [cli_args] + + .. tab-item:: isaaclab.sh / isaaclab.bat + + .. code-block:: bash + + ./isaaclab.sh -p -m debugpy --listen 3000 --wait-for-client -c "from isaaclab.cli import cli; cli()" [cli_args] + +3. In VS Code, select the ``Python: Debugger Attach`` configuration from the Run and Debug panel + and press the green play button or ``F5``. VS Code will connect to the debugpy server + running on ``localhost:3000``. + +Configuring the Python interpreter +---------------------------------- + +The setup command records the interpreter that ran it in +``.vscode/settings.json``. For example, a uv source checkout on Linux uses: + +.. code-block:: json + + { + "python.defaultInterpreterPath": "/path/to/IsaacLab/.venv/bin/python", + } + +The editor selection takes precedence over this default. If you change +environments, rerun setup and select the new interpreter from the status bar or +with **Python: Select Interpreter** in the command palette. + +For more information about selecting a Python interpreter, see the +`VS Code documentation `_. + + +Setting up formatting and linting +--------------------------------- + +We use `ruff `_ as a formatter and linter. +These are configured in the ``.vscode/settings.json`` file: + +.. code-block:: json + + { + "ruff.configuration": "${workspaceFolder}/pyproject.toml", + } + +The ruff linter will show warnings and errors in your code to help you follow Python best practices and the project's coding standards. diff --git a/docs/source/developer-tools/template_generator.rst b/docs/source/developer-tools/template_generator.rst index c69288396663..4d426df9d49d 100644 --- a/docs/source/developer-tools/template_generator.rst +++ b/docs/source/developer-tools/template_generator.rst @@ -242,11 +242,27 @@ External projects should build their environment harness from public APIs and maintain project-local fixtures. Copying ``env_test_utils.py`` into a project is vendoring it, so the project must track upstream changes to that copy. -To configure VS Code, run the generated setup task or invoke it directly: +To configure VS Code or Cursor, run the generated setup task or invoke it directly: .. code-block:: bash - uv run python .vscode/tools/setup_vscode.py + uv run isaaclab --editor + +The command selects the active interpreter and creates a git-ignored +``pyrightconfig.json``. This child configuration inherits the checked-in +Pyright policy from ``pyproject.toml`` and adds the generated project's +``src`` import root, installed Isaac Lab packages, and any discovered Isaac +Sim extensions. When using the ``isaacsim`` extra, include it while generating +the configuration: + +.. code-block:: bash + + uv run --extra isaacsim isaaclab --editor + +In VS Code, use Pylance and select the interpreter that ran the setup command. +In Cursor, install the ``detachhead.basedpyright`` extension instead of Pylance, +select the same interpreter, and reload the window. Both language servers read +the generated ``pyrightconfig.json``. Create an internal task ----------------------- diff --git a/docs/source/overview/developer-guide/index.rst b/docs/source/overview/developer-guide/index.rst index d006f1f8663e..ab96af49054b 100644 --- a/docs/source/overview/developer-guide/index.rst +++ b/docs/source/overview/developer-guide/index.rst @@ -1,16 +1,11 @@ Developer's Guide ================= -For development, we suggest using `Microsoft Visual Studio Code -(VSCode) `__. This is also suggested by -NVIDIA Omniverse and there exists tutorials on how to `debug Omniverse -extensions `__ -using VSCode. +Resources for developing and contributing to Isaac Lab. .. toctree:: :maxdepth: 1 - VS Code repo_structure development agent_skills diff --git a/docs/source/overview/developer-guide/vs_code.rst b/docs/source/overview/developer-guide/vs_code.rst deleted file mode 100644 index 23ea18956bdc..000000000000 --- a/docs/source/overview/developer-guide/vs_code.rst +++ /dev/null @@ -1,121 +0,0 @@ -.. _setup-vs-code: - -Setting up Visual Studio Code ------------------------------ - -**This is optional. You do not need to use VScode to use Isaac Lab** - -`Visual Studio Code `_ has proven an invaluable tool for the development of Isaac Lab. The Isaac Lab repository includes the VSCode files for setting up your development environment. These are included in the ``.vscode`` directory and include the following files: - -.. code-block:: bash - - .vscode - ├── tools - │   ├── launch.template.json - │   ├── settings.template.json - │   └── setup_vscode.py - ├── extensions.json - ├── launch.json # <- this is generated by setup_vscode.py - ├── settings.json # <- this is generated by setup_vscode.py - └── tasks.json - - -.. attention:: - - The following instructions on setting up Visual Studio Code only work with - :ref:`Isaac Sim Binaries Installation ` and not with - :ref:`Python Environment with Isaac Sim `. - - -To setup the IDE, please follow these instructions: - -1. Open the ``IsaacLab`` directory on Visual Studio Code IDE -2. Run VSCode `Tasks `__, by - pressing ``Ctrl+Shift+P``, selecting ``Tasks: Run Task`` and running the - ``setup_python_env`` in the drop down menu. - - .. image:: ../../_static/vscode_tasks.png - :width: 600px - :align: center - :alt: VSCode Tasks - - -.. note:: - If this is your first time running tasks in VS Code, you may be prompted to select how to handle warnings. Simply follow - the prompts until the task window closes. - -If everything executes correctly, it should create the following files: - -* ``.vscode/launch.json``: Contains the launch configurations for debugging python code. -* ``.vscode/settings.json``: Contains the settings for the python interpreter and the python environment. - -For more information on VSCode support for Omniverse, please refer to the -following links: - -* `Isaac Sim VSCode support `__ - - -Attach to a Running ``debugpy`` Session ---------------------------------------- - -The generated ``.vscode/launch.json`` includes a ``Python: Debugger Attach`` -configuration that starts to listen on port ``localhost:3000`` for the debugpy session. - -To use it: - -1. Set your breakpoints. -2. Run your code under debugpy like so: - - .. tab-set:: - - .. tab-item:: uv (Recommended) - - .. code-block:: bash - - uv run python -m debugpy --listen 3000 --wait-for-client -c "from isaaclab.cli import cli; cli()" [cli_args] - - .. tab-item:: isaaclab.sh / isaaclab.bat - - .. code-block:: bash - - ./isaaclab.sh -p -m debugpy --listen 3000 --wait-for-client -c "from isaaclab.cli import cli; cli()" [cli_args] - -3. In VS Code, select the ``Python: Debugger Attach`` configuration from the Run and Debug panel - and press the green play button or ``F5``. VS Code will connect to the debugpy server - running on ``localhost:3000``. - -Configuring the python interpreter ----------------------------------- - -In the provided configuration, we set the default python interpreter to use the -python executable provided by Omniverse. This is specified in the -``.vscode/settings.json`` file: - -.. code-block:: json - - { - "python.defaultInterpreterPath": "${workspaceFolder}/_isaac_sim/python.sh", - } - -If you want to use a different python interpreter (for instance, from your conda or uv environment), -you need to change the python interpreter used by selecting and activating the python interpreter -of your choice in the bottom left corner of VSCode, or opening the command palette (``Ctrl+Shift+P``) -and selecting ``Python: Select Interpreter``. - -For more information on how to set python interpreter for VSCode, please -refer to the `VSCode documentation `_. - - -Setting up formatting and linting ---------------------------------- - -We use `ruff `_ as a formatter and linter. -These are configured in the ``.vscode/settings.json`` file: - -.. code-block:: json - - { - "ruff.configuration": "${workspaceFolder}/pyproject.toml", - } - -The ruff linter will show warnings and errors in your code to help you follow Python best practices and the project's coding standards. diff --git a/docs/source/setup/installation/index.rst b/docs/source/setup/installation/index.rst index a5d056c6c16f..a153cd81613f 100644 --- a/docs/source/setup/installation/index.rst +++ b/docs/source/setup/installation/index.rst @@ -771,16 +771,17 @@ The first launch downloads Isaac Sim extensions and can take more than ten minut you to accept the NVIDIA Omniverse EULA; set ``OMNI_KIT_ACCEPT_EULA=yes`` for a non-interactive environment. Run a project script with ``python my_script.py``. -Generate VS Code settings for the current workspace with: +Generate VS Code or Cursor settings for the current workspace with: .. code-block:: bash - python -m isaaclab --generate-vscode-settings + uv run isaaclab --editor .. warning:: - This command generates ``.vscode/settings.json`` in the workspace. If the file already exists, - it asks before overwriting it. + This command generates ``.vscode/settings.json`` and ``pyrightconfig.json`` in the workspace. + The Pyright configuration inherits an existing ``[tool.pyright]`` table and adds paths discovered + from the active Python environment. .. _installation-method-binary: .. _isaaclab-binaries-installation: diff --git a/docs/source/setup/quickstart.rst b/docs/source/setup/quickstart.rst index 78c8bf563cb8..0dcfcc36a08b 100644 --- a/docs/source/setup/quickstart.rst +++ b/docs/source/setup/quickstart.rst @@ -36,6 +36,26 @@ Training outputs, including checkpoints, are saved under ``logs/``. Add uv run isaaclab train --help +Configure an editor (optional) +------------------------------ + +To enable import completion and debugging in VS Code or Cursor, generate the +machine-local editor configuration from the repository root: + +.. code-block:: bash + + uv run isaaclab --editor + +If you use the ``isaacsim`` extra, include it so the command can discover the +Isaac Sim extensions: + +.. code-block:: bash + + uv run --extra isaacsim isaaclab --editor + +VS Code uses Pylance. Cursor users should install basedpyright instead. See +:ref:`setup-vs-code` for complete editor and troubleshooting instructions. + .. The quickstart media is generated by tools/docs/media/generate_quickstart.sh. .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/quickstart_task_categories.gif diff --git a/pyproject.toml b/pyproject.toml index d868dff524f0..7265a7cd6d0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -328,6 +328,11 @@ exclude = [ ".vscode", ] +# Make the in-repo packages resolvable by the language server (Pylance / basedpyright) +# without an editable install, so imports like ``isaaclab.assets`` work out of the box. +# The glob keeps new source packages discoverable without maintaining a duplicate package list. +extraPaths = ["source/*"] + typeCheckingMode = "basic" pythonVersion = "3.12" pythonPlatform = "Linux" diff --git a/source/isaaclab/changelog.d/mhaiderbhai-fix-pyright-editor-setup.rst b/source/isaaclab/changelog.d/mhaiderbhai-fix-pyright-editor-setup.rst new file mode 100644 index 000000000000..570bda8b810c --- /dev/null +++ b/source/isaaclab/changelog.d/mhaiderbhai-fix-pyright-editor-setup.rst @@ -0,0 +1,6 @@ +Fixed +^^^^^ + +* Fixed VS Code and Cursor import resolution for source, editable, wheel, and Isaac Sim binaries installations. + Replaced ``isaaclab --vscode`` and ``python -m isaaclab --generate-vscode-settings`` with + ``uv run isaaclab --editor``. diff --git a/source/isaaclab/isaaclab/__main__.py b/source/isaaclab/isaaclab/__main__.py index 5ea66bbe64c3..eafe689b33e7 100644 --- a/source/isaaclab/isaaclab/__main__.py +++ b/source/isaaclab/isaaclab/__main__.py @@ -3,153 +3,8 @@ # # SPDX-License-Identifier: BSD-3-Clause -import argparse -import os -import re -import sys -import textwrap - -import tomllib - -from isaaclab.paths import ISAACLAB_ROOT - -VSCODE_SETTINGS_TEMPLATE = """ -{ - "editor.rulers": [120], - - // Enables python language server (seems to work slightly better than jedi) - "python.languageServer": "Pylance", - "python.jediEnabled": false, - - // Those paths are automatically filled by isaaclab (see: 'python -m isaaclab --help') - "python.defaultInterpreterPath": "PYTHON.DEFAULTINTERPRETERPATH", - "python.analysis.extraPaths": [ - PYTHON.ANALYSIS.EXTRAPATHS - ], - - // Use "black" as a formatter - "python.formatting.provider": "black", - "python.formatting.blackArgs": ["--line-length", "120"], - - // Use "flake8" for linting - "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": true, -} -""" - - -def generate_vscode_settings(): - def _mock_python_modules(ext_path: str, ext_name: str) -> None: - # parse config/extension.toml - cprint(f" |-- Parsing extension config ({ext_path})") - config_path = os.path.join(ext_path, "config", "extension.toml") - try: - with open(config_path, "rb") as f: - config = tomllib.load(f) - except Exception as e: - cprint(f" | |-- [Warning] {e}") - return - # get python modules - for item in config.get("python", {}).get("module", []): - if list(item.keys()) == ["name"]: - # skip tests - if item.get("name", "").endswith(".tests"): - continue - # mock __init__.py for each submodule (if not exists) - submodule_path = ext_path - for submodule in item.get("name", "").split("."): - init_path = os.path.join(submodule_path, "__init__.py") - if not os.path.isfile(init_path): - try: - cprint(f" |-- Mocking {init_path}") - with open(init_path, "w") as f: - f.write("# Generated by 'isaaclab' package") - except Exception as e: - cprint(f" | |-- [Warning] {e}") - continue - submodule_path = os.path.join(submodule_path, submodule) - - def _get_paths(base_path: str, mock_python_modules: bool = False) -> list[str]: - paths = [] - if os.path.isdir(base_path): - for folder in os.listdir(base_path): - folder_path = os.path.join(base_path, folder) - if os.path.isdir(folder_path): - paths.append(folder_path) - cprint(f"Registering extension: {folder_path}") - if mock_python_modules: - _mock_python_modules(folder_path, re.split(r"-\d+", folder)[0]) - return paths - - try: - import omni.kit_app # importing 'omni.kit_app' will bootstrap kernel - - kit_path = os.path.dirname(os.path.abspath(os.path.realpath(omni.kit_app.__file__))) - except ModuleNotFoundError: - print("Unable to find 'omniverse-kit' package") - # exit() - try: - import isaacsim - - isaacsim_path = os.path.dirname(os.path.abspath(os.path.realpath(isaacsim.__file__))) - except ModuleNotFoundError: - print("Unable to find 'isaacsim' package") - # exit() - - cwd = os.getcwd() - vscode_settings_path = os.path.join(cwd, ".vscode", "settings.json") - # check if .vscode/settings.json exists - if os.path.exists(vscode_settings_path): - print(f"VS Code settings already exists: {vscode_settings_path}") - if input("Overwrite? (y/N): ").lower() not in ["y", "yes"]: - print("Cancelled: VS Code settings not overwritten") - return - - # get extensions paths - extensions_paths = [] - # - omniverse-kit - folder_path = os.path.join(kit_path, "kernel", "py") - if os.path.isdir(folder_path): - extensions_paths.append(folder_path) - for folder in ["exts", "extscore"]: - extensions_paths.extend(_get_paths(os.path.join(kit_path, folder), mock_python_modules=True)) - # - isaacsim - for folder in ["exts", "extscache", "extsDeprecated", "extsUser"]: - extensions_paths.extend(_get_paths(os.path.join(isaacsim_path, folder), mock_python_modules=True)) - # - isaaclab - isaaclab_path = str(ISAACLAB_ROOT) - for folder in ["source"]: - extensions_paths.extend(_get_paths(os.path.join(isaaclab_path, folder), mock_python_modules=True)) - - # update 'python.defaultInterpreterPath' - template = VSCODE_SETTINGS_TEMPLATE[:] - template = template.replace("PYTHON.DEFAULTINTERPRETERPATH", sys.executable) - - # update 'python.analysis.extraPaths' - content = "\n".join([f'"{path}",' for path in extensions_paths]) - content = textwrap.indent(content, prefix=" " * 8)[8:] - template = template.replace("PYTHON.ANALYSIS.EXTRAPATHS", content) - - # create .vscode/settings.json - os.makedirs(os.path.join(cwd, ".vscode"), exist_ok=True) - with open(vscode_settings_path, "w") as f: - f.write(template) - print("VS Code settings generated at", vscode_settings_path) - - def main(): - """Run the installed Isaac Lab CLI while preserving the legacy VS Code option.""" - if len(sys.argv) > 1 and sys.argv[1] == "--generate-vscode-settings": - parser = argparse.ArgumentParser() - parser.add_argument("--generate-vscode-settings", action="store_true", help="Generate VS Code settings.") - parser.add_argument("--verbose", action="store_true", help="Print discovered extension paths.") - args = parser.parse_args() - - global cprint - cprint = print if args.verbose else lambda *args, **kwargs: None - generate_vscode_settings() - return - + """Run the Isaac Lab CLI.""" from isaaclab.cli import cli cli() diff --git a/source/isaaclab/isaaclab/cli/__init__.py b/source/isaaclab/isaaclab/cli/__init__.py index 2961bdd8e83b..cbe278b3fcce 100644 --- a/source/isaaclab/isaaclab/cli/__init__.py +++ b/source/isaaclab/isaaclab/cli/__init__.py @@ -19,11 +19,11 @@ from .commands.misc import ( command_build_docs, command_build_isaacsim, + command_editor, command_new, command_run_docker, command_run_isaacsim, command_test, - command_vscode_settings, ) from .utils import ( ISAACLAB_ROOT, @@ -244,10 +244,9 @@ def cli() -> None: help="Run the docker container helper script (docker/container.sh).", ) parser.add_argument( - "-v", - "--vscode", - action="store_true", - help="Generate the VSCode settings file from template.", + "--editor", + nargs=argparse.REMAINDER, + help="Generate editor settings and import paths for the current workspace.", ) parser.add_argument( "-d", @@ -307,8 +306,8 @@ def cli() -> None: elif args.isaacsim_source: command_build_isaacsim(args.isaacsim_source) - elif args.vscode: - command_vscode_settings() + elif args.editor is not None: + command_editor(args.editor) elif args.docs: command_build_docs() diff --git a/source/isaaclab/isaaclab/cli/commands/install.py b/source/isaaclab/isaaclab/cli/commands/install.py index b30bcfeaec5a..436f3b703135 100644 --- a/source/isaaclab/isaaclab/cli/commands/install.py +++ b/source/isaaclab/isaaclab/cli/commands/install.py @@ -26,7 +26,7 @@ print_warning, run_command, ) -from .misc import command_vscode_settings +from .misc import command_editor _PACKAGE_INDEX_RETRIES = "12" _PACKAGE_INSTALL_RETRY_ATTEMPTS = 3 @@ -1356,6 +1356,6 @@ def append_submodules_once(package_dirs: tuple[str, ...]) -> None: if saved_pythonpath is not None: os.environ["PYTHONPATH"] = saved_pythonpath - # Install vscode update unless we're in docker. + # Update editor settings unless we're in Docker. if not (os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv")): - command_vscode_settings() + command_editor([], project_dir=ISAACLAB_ROOT) diff --git a/source/isaaclab/isaaclab/cli/commands/misc.py b/source/isaaclab/isaaclab/cli/commands/misc.py index 612f1e1e6cb7..d456ee20ca50 100644 --- a/source/isaaclab/isaaclab/cli/commands/misc.py +++ b/source/isaaclab/isaaclab/cli/commands/misc.py @@ -5,6 +5,7 @@ """Misc commands""" +import argparse import platform import shutil import sys @@ -17,7 +18,6 @@ is_windows, print_error, print_info, - print_warning, run_command, run_python_command, ) @@ -61,20 +61,26 @@ def command_test(test_args: list[str]) -> None: run_python_command("-m", ["pytest", str(ISAACLAB_ROOT / "tools")] + test_args) -def command_vscode_settings() -> None: - """Update the vscode settings from template and Isaac Sim settings""" +def command_editor(editor_args: list[str], project_dir: Path | None = None) -> None: + """Generate editor settings and import paths for a workspace. - print_info("Setting up vscode settings...") + Args: + editor_args: Editor setup command arguments. + project_dir: Workspace root. Defaults to the current directory. + """ + parser = argparse.ArgumentParser(prog="isaaclab --editor", description="Set up editor paths and settings.") + parser.add_argument("--isaac_path", help="Absolute path to the Isaac Sim installation.") + parser.add_argument("--verbose", action="store_true", help="Print discovered extension paths.") + args = parser.parse_args(editor_args) - # Path to setup_vscode.py. - setup_vscode_script = ISAACLAB_ROOT / ".vscode" / "tools" / "setup_vscode.py" + # The installation CLI must start before Isaac Lab's runtime dependencies are installed. + from ...utils.editor import setup_editor - # Check if the file exists before attempting to run it. - if setup_vscode_script.exists(): - run_python_command(setup_vscode_script, []) - print_info("VS Code settings generated successfully.") - else: - print_warning("Unable to find the script 'setup_vscode.py'. Aborting vscode settings setup.") + print_info("Setting up editor paths and settings...") + try: + setup_editor(project_dir or Path.cwd(), isaac_path=args.isaac_path, verbose=args.verbose) + except ValueError as error: + parser.error(str(error)) def command_build_docs() -> None: @@ -189,7 +195,7 @@ def _resolve_isaacsim_release_dir(isaacsim_root: Path) -> Path: def _repoint_source_build_prebundles() -> None: """Keep Isaac Sim's prebundled packages from shadowing the active environment.""" - # ``install`` imports ``command_vscode_settings`` from this module, so defer this import until + # ``install`` imports ``command_editor`` from this module, so defer this import until # both command modules are initialized. Reuse the same protection as the legacy installer. from .install import _repoint_prebundle_packages diff --git a/source/isaaclab/isaaclab/utils/editor.py b/source/isaaclab/isaaclab/utils/editor.py new file mode 100644 index 000000000000..cd60f651a374 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/editor.py @@ -0,0 +1,235 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Utilities for generating editor import paths for Isaac Lab projects.""" + +import importlib.metadata +import importlib.util +import json +import os +import pathlib +import re +import subprocess +import sys + +_DEFAULT_VSCODE_SETTINGS_TEMPLATE = """ +{ + "editor.rulers": [120], + + "python.languageServer": "Pylance", + "python.jediEnabled": false, + "python.defaultInterpreterPath": "", + + "python.formatting.provider": "black", + "python.formatting.blackArgs": ["--line-length", "120"], + + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + + "[python]": { + "editor.tabSize": 4 + } +} +""" + + +def setup_editor(project_dir: pathlib.Path, isaac_path: str | None = None, verbose: bool = False) -> None: + """Generate editor settings and Pyright import paths for a project. + + Args: + project_dir: Project root where the editor configuration is written. + isaac_path: Explicit Isaac Sim installation path, or None to discover it. + verbose: Whether to print every generated Pyright search path. + """ + isaacsim_dir = resolve_isaacsim_dir(project_dir, isaac_path) + extra_paths = build_extra_paths(project_dir, isaacsim_dir) + write_pyright_config(project_dir, extra_paths) + if verbose: + for path in extra_paths: + print(f"Registered Pyright search path: {path}") + + vscode_dir = project_dir / ".vscode" + settings_template = vscode_dir / "tools" / "settings.template.json" + if settings_template.is_file(): + settings = settings_template.read_text(encoding="utf-8") + settings_source = settings_template.as_posix() + else: + settings = _DEFAULT_VSCODE_SETTINGS_TEMPLATE + settings_source = "Isaac Lab defaults" + settings = _overwrite_default_python_interpreter(settings, isaacsim_dir) + + vscode_dir.mkdir(parents=True, exist_ok=True) + header = ( + "// This file is automatically generated by `isaaclab --editor`.\n" + "// Do not edit this file directly.\n" + f"// Generated from: {settings_source}\n" + ) + settings_path = vscode_dir / "settings.json" + settings_path.write_text(header + settings, encoding="utf-8") + + launch_path = vscode_dir / "launch.json" + launch_template = vscode_dir / "tools" / "launch.template.json" + if not launch_path.exists() and launch_template.is_file(): + launch_header = header.replace(settings_source, launch_template.as_posix()) + launch_path.write_text(launch_header + launch_template.read_text(encoding="utf-8"), encoding="utf-8") + + print(f"Editor settings generated at {settings_path}") + print(f"Pyright configuration generated at {project_dir / 'pyrightconfig.json'}") + + +def resolve_isaacsim_dir(project_dir: pathlib.Path, isaac_path: str | None = None) -> pathlib.Path | None: + """Resolve the Isaac Sim installation directory. + + Args: + project_dir: Project root containing an optional ``_isaac_sim`` link. + isaac_path: Explicit Isaac Sim path, or None to discover the installation. + + Returns: + The resolved installation directory, or None if Isaac Sim is unavailable. + + Raises: + ValueError: If an explicit path does not identify a directory. + """ + if isaac_path: + explicit_path = pathlib.Path(isaac_path).expanduser() + if not _is_isaacsim_dir(explicit_path): + raise ValueError(f"Not an Isaac Sim directory (missing .vscode/settings.json): {explicit_path}") + return explicit_path.resolve() + + env_path = os.environ.get("ISAAC_PATH") + if env_path and _is_isaacsim_dir(pathlib.Path(env_path)): + return pathlib.Path(env_path).resolve() + + probe = subprocess.run( + [sys.executable, "-c", "import isaacsim, os; print(os.environ.get('ISAAC_PATH', ''))"], + capture_output=True, + text=True, + check=False, + stdin=subprocess.DEVNULL, + ) + for line in reversed(probe.stdout.splitlines()): + candidate = pathlib.Path(line.strip()).expanduser() + if line.strip() and _is_isaacsim_dir(candidate): + return candidate.resolve() + + fallback = project_dir / "_isaac_sim" + return fallback.resolve() if _is_isaacsim_dir(fallback) else None + + +def read_isaacsim_extra_paths(isaacsim_dir: pathlib.Path | None) -> list[pathlib.Path]: + """Read Isaac Sim's Python extension paths. + + Args: + isaacsim_dir: Isaac Sim installation directory, or None. + + Returns: + Absolute extension search paths. + """ + if isaacsim_dir is None: + print("[WARN] Isaac Sim was not found; simulator extension paths were not added.") + return [] + + settings_file = isaacsim_dir / ".vscode" / "settings.json" + if not settings_file.is_file(): + print(f"[WARN] Isaac Sim VS Code settings were not found: {settings_file}") + return [] + + settings = settings_file.read_text(encoding="utf-8") + match = re.search(r'"python\.analysis\.extraPaths"\s*:\s*\[(.*?)\]', settings, flags=re.DOTALL) + if match is None: + print(f"[WARN] python.analysis.extraPaths was not found in {settings_file}") + return [] + + paths = [] + for encoded_path in re.findall(r'"((?:\\.|[^"\\])*)"', match.group(1)): + path = pathlib.Path(json.loads(f'"{encoded_path}"')) + paths.append(path if path.is_absolute() else isaacsim_dir / path) + return paths + + +def find_isaaclab_package_paths() -> list[pathlib.Path]: + """Find Isaac Lab package roots visible to the active interpreter. + + Returns: + Import roots for installed Isaac Lab packages. + """ + paths = [] + package_names = sorted(name for name in importlib.metadata.packages_distributions() if name.startswith("isaaclab")) + for package_name in package_names: + spec = importlib.util.find_spec(package_name) + if spec is None: + continue + if spec.submodule_search_locations: + paths.extend(pathlib.Path(location).parent for location in spec.submodule_search_locations) + elif spec.origin: + paths.append(pathlib.Path(spec.origin).parent) + return paths + + +def build_extra_paths(project_dir: pathlib.Path, isaacsim_dir: pathlib.Path | None) -> list[str]: + """Build Pyright search paths for simulator, local, and installed packages. + + Args: + project_dir: Project root used to discover local ``source/*`` or ``src`` packages. + isaacsim_dir: Isaac Sim installation directory, or None. + + Returns: + Deduplicated paths, relative to the project where practical. + """ + paths = read_isaacsim_extra_paths(isaacsim_dir) + source_dir = project_dir / "source" + if source_dir.is_dir(): + paths.extend(path for path in sorted(source_dir.iterdir()) if path.is_dir()) + src_dir = project_dir / "src" + if src_dir.is_dir(): + paths.append(src_dir) + paths.extend(find_isaaclab_package_paths()) + + formatted_paths = [] + seen = set() + resolved_project_dir = project_dir.resolve() + for path in paths: + resolved_path = path.resolve() + try: + formatted_path = resolved_path.relative_to(resolved_project_dir).as_posix() + except ValueError: + formatted_path = resolved_path.as_posix() + if formatted_path not in seen: + seen.add(formatted_path) + formatted_paths.append(formatted_path) + return formatted_paths + + +def write_pyright_config(project_dir: pathlib.Path, extra_paths: list[str]): + """Write a machine-local Pyright configuration that preserves project policy. + + Args: + project_dir: Project root where the configuration is written. + extra_paths: Additional import search paths. + """ + config: dict[str, object] = {"extraPaths": extra_paths} + if (project_dir / "pyproject.toml").is_file(): + config["extends"] = "./pyproject.toml" + (project_dir / "pyrightconfig.json").write_text(json.dumps(config, indent=4) + "\n", encoding="utf-8") + + +def _is_isaacsim_dir(path: pathlib.Path) -> bool: + """Check whether a directory contains the settings used for extension discovery.""" + return path.is_dir() and (path / ".vscode" / "settings.json").is_file() + + +def _overwrite_default_python_interpreter(settings: str, isaacsim_dir: pathlib.Path | None) -> str: + """Set the editor's default Python interpreter.""" + python_exe = pathlib.Path(sys.executable) + if "kit/python/bin/python3" in python_exe.as_posix() and isaacsim_dir is not None: + wrapper = isaacsim_dir / "python.sh" + if wrapper.is_file(): + python_exe = wrapper + return re.sub( + r'"python\.defaultInterpreterPath": ".*?"', + f'"python.defaultInterpreterPath": "{python_exe.as_posix()}"', + settings, + flags=re.DOTALL, + ) diff --git a/source/isaaclab/test/cli/test_install_command_parsing.py b/source/isaaclab/test/cli/test_install_command_parsing.py index f456c638e682..a24f6df1cd1d 100644 --- a/source/isaaclab/test/cli/test_install_command_parsing.py +++ b/source/isaaclab/test/cli/test_install_command_parsing.py @@ -185,7 +185,7 @@ def test_ov_selector_installs_matching_root_extra(selector, expected_extra): f"{_INSTALL_MODULE}._maybe_uninstall_prebundled_torch", f"{_INSTALL_MODULE}._ensure_pink_ik_dependencies_installed", f"{_INSTALL_MODULE}._repoint_prebundle_packages", - f"{_INSTALL_MODULE}.command_vscode_settings", + f"{_INSTALL_MODULE}.command_editor", f"{_INSTALL_MODULE}.get_pip_command", f"{_INSTALL_MODULE}.extract_python_exe", # run_command is called directly inside command_install for pip/setuptools upgrades. diff --git a/source/isaaclab/test/cli/test_installed_workflow_entrypoints.py b/source/isaaclab/test/cli/test_installed_workflow_entrypoints.py index 9a8951730e8f..e43cbcf5fe8c 100644 --- a/source/isaaclab/test/cli/test_installed_workflow_entrypoints.py +++ b/source/isaaclab/test/cli/test_installed_workflow_entrypoints.py @@ -7,6 +7,7 @@ from __future__ import annotations +import subprocess import sys from unittest import mock @@ -20,6 +21,18 @@ pytestmark = pytest.mark.unit +def test_cli_import_does_not_require_runtime_dependencies(): + """The installation CLI must load before core runtime dependencies are installed.""" + result = subprocess.run( + [sys.executable, "-c", 'import sys; sys.modules["lazy_loader"] = None; import isaaclab.cli'], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + def test_resolves_partial_source_checkout_root(tmp_path): """Source root resolution must not require resources copied by later Docker layers.""" package_root = tmp_path / "source" / "isaaclab" / "isaaclab" @@ -38,15 +51,22 @@ def test_top_level_compatibility_api_is_preserved(): main.assert_called_once_with() -def test_legacy_vscode_option_uses_compatibility_dispatcher(): - """The installed entry point must continue to recognize the legacy VS Code option.""" +def test_editor_option_uses_cli_dispatcher(): + """The installed CLI must forward editor-specific arguments to the editor command.""" with ( - mock.patch.object(sys, "argv", ["isaaclab", "--generate-vscode-settings"]), - mock.patch.object(package_main, "generate_vscode_settings") as generate, + mock.patch.object(sys, "argv", ["isaaclab", "--editor", "--isaac_path", "/sim", "--verbose"]), + mock.patch.object(cli, "command_editor") as editor, ): - package_main.main() + cli.cli() - generate.assert_called_once_with() + editor.assert_called_once_with(["--isaac_path", "/sim", "--verbose"]) + + +@pytest.mark.parametrize("option", ["--vscode", "--generate-vscode-settings"]) +def test_removed_editor_options_are_rejected(option): + """Removed editor setup options must not remain as hidden compatibility paths.""" + with mock.patch.object(sys, "argv", ["isaaclab", option]), pytest.raises(SystemExit, match="2"): + cli.cli() @pytest.mark.parametrize( diff --git a/source/isaaclab_rl/changelog.d/mhaiderbhai-vscode-pyright-tests.skip b/source/isaaclab_rl/changelog.d/mhaiderbhai-vscode-pyright-tests.skip new file mode 100644 index 000000000000..3259ba2642e0 --- /dev/null +++ b/source/isaaclab_rl/changelog.d/mhaiderbhai-vscode-pyright-tests.skip @@ -0,0 +1 @@ +The template regression suite covers editor configuration generation. diff --git a/source/isaaclab_rl/test/test_template_generator.py b/source/isaaclab_rl/test/test_template_generator.py index 65bc3d786ae1..91b287c10623 100644 --- a/source/isaaclab_rl/test/test_template_generator.py +++ b/source/isaaclab_rl/test/test_template_generator.py @@ -7,6 +7,7 @@ import ast import importlib.util +import json import pkgutil import subprocess import sys @@ -17,6 +18,8 @@ import pytest import tomllib +from isaaclab.utils import editor as editor_utils + ROOT_DIR = Path(__file__).resolve().parents[3] TEMPLATE_TOOL_DIR = ROOT_DIR / "tools" / "template" sys.path.insert(0, str(TEMPLATE_TOOL_DIR)) @@ -302,11 +305,84 @@ def test_external_project_uses_src_layout_and_installed_isaaclab_commands(tmp_pa "build-backend": "uv_build", } assert package["project"]["dependencies"] == ["isaaclab[rsl-rl,skrl]"] + assert package["tool"]["pyright"] == { + "include": ["src", "scripts", "tests"], + "exclude": ["**/__pycache__", "**/logs", ".git", ".venv", ".vscode"], + "typeCheckingMode": "basic", + } assert package["project"]["entry-points"]["isaaclab.tasks"] == {project_name: f"{project_name}.tasks"} assert package["tool"]["uv"]["build-backend"]["module-name"] == project_name assert (project_dir / "src" / project_name / "__init__.py").is_file() assert not (project_dir / "source").exists() assert {path.name for path in (project_dir / "scripts").iterdir()} == {"list_envs.py"} + assert "pyrightconfig.json" in (project_dir / ".gitignore").read_text() + assert ( + "python.analysis.extraPaths" not in (project_dir / ".vscode" / "tools" / "settings.template.json").read_text() + ) + assert not (project_dir / ".vscode" / "tools" / "setup_vscode.py").exists() + assert "uv run isaaclab --editor" in (project_dir / ".vscode" / "tasks.json").read_text() + + +def test_editor_setup_combines_simulator_local_and_installed_paths(tmp_path, monkeypatch): + """The generated Pyright child config must preserve project policy and cover every installation mode.""" + project_dir = tmp_path / "project" + (project_dir / "source" / "local_package").mkdir(parents=True) + (project_dir / "src" / "generated_project").mkdir(parents=True) + (project_dir / "pyproject.toml").write_text('[tool.pyright]\ntypeCheckingMode = "basic"\n') + isaacsim_dir = tmp_path / "isaacsim" + (isaacsim_dir / ".vscode").mkdir(parents=True) + (isaacsim_dir / ".vscode" / "settings.json").write_text( + '{"python.analysis.extraPaths": ["exts/isaacsim.core.api", "extscache/omni.kit.foo"]}' + ) + installed_root = tmp_path / "editable" / "isaaclab" + installed_root.mkdir(parents=True) + + monkeypatch.setattr(editor_utils, "find_isaaclab_package_paths", lambda: [installed_root]) + + extra_paths = editor_utils.build_extra_paths(project_dir, isaacsim_dir) + editor_utils.write_pyright_config(project_dir, extra_paths) + config = json.loads((project_dir / "pyrightconfig.json").read_text()) + + assert config["extends"] == "./pyproject.toml" + assert config["extraPaths"] == [ + (isaacsim_dir / "exts" / "isaacsim.core.api").as_posix(), + (isaacsim_dir / "extscache" / "omni.kit.foo").as_posix(), + "source/local_package", + "src", + installed_root.as_posix(), + ] + + +def test_editor_setup_uses_workspace_templates(tmp_path, monkeypatch): + """Editor setup must generate workspace files without a project-local Python wrapper.""" + project_dir = tmp_path / "project" + tools_dir = project_dir / ".vscode" / "tools" + tools_dir.mkdir(parents=True) + (tools_dir / "settings.template.json").write_text('{"python.defaultInterpreterPath": ""}\n') + (tools_dir / "launch.template.json").write_text('{"version": "0.2.0", "configurations": []}\n') + isaacsim_dir = tmp_path / "isaacsim" + (isaacsim_dir / ".vscode").mkdir(parents=True) + (isaacsim_dir / ".vscode" / "settings.json").write_text('{"python.analysis.extraPaths": []}') + monkeypatch.setattr(editor_utils, "find_isaaclab_package_paths", lambda: []) + + editor_utils.setup_editor(project_dir, isaac_path=str(isaacsim_dir)) + + settings = (project_dir / ".vscode" / "settings.json").read_text() + assert "automatically generated by `isaaclab --editor`" in settings + assert Path(sys.executable).as_posix() in settings + assert (project_dir / ".vscode" / "launch.json").is_file() + assert not (tools_dir / "setup_vscode.py").exists() + assert json.loads((project_dir / "pyrightconfig.json").read_text()) == {"extraPaths": []} + + +@pytest.mark.parametrize("path_exists", [False, True]) +def test_editor_setup_rejects_invalid_explicit_isaac_sim_path(tmp_path, path_exists): + """An invalid user-selected installation must not silently select a different Isaac Sim.""" + invalid_path = tmp_path / "invalid" + if path_exists: + invalid_path.mkdir() + with pytest.raises(ValueError, match="Not an Isaac Sim directory"): + editor_utils.resolve_isaacsim_dir(tmp_path, str(invalid_path)) def _all_libraries() -> list[dict]: diff --git a/tools/template/templates/external/.gitignore b/tools/template/templates/external/.gitignore index d7e3d459d3d1..d8f5bdf5325e 100644 --- a/tools/template/templates/external/.gitignore +++ b/tools/template/templates/external/.gitignore @@ -14,3 +14,4 @@ wandb/ .vscode/settings.json .vscode/launch.json +pyrightconfig.json diff --git a/tools/template/templates/external/.vscode/.gitignore b/tools/template/templates/external/.vscode/.gitignore index 10b0af342ce3..e0a8bea56c6f 100644 --- a/tools/template/templates/external/.vscode/.gitignore +++ b/tools/template/templates/external/.vscode/.gitignore @@ -1,7 +1,6 @@ # Note: These files are kept for development purposes only. !tools/launch.template.json !tools/settings.template.json -!tools/setup_vscode.py !extensions.json !tasks.json diff --git a/tools/template/templates/external/.vscode/tasks.json b/tools/template/templates/external/.vscode/tasks.json index db7bd446edf4..f7751b119f55 100644 --- a/tools/template/templates/external/.vscode/tasks.json +++ b/tools/template/templates/external/.vscode/tasks.json @@ -4,7 +4,7 @@ { "label": "setup_python_env", "type": "shell", - "command": "uv run python ${workspaceFolder}/.vscode/tools/setup_vscode.py", + "command": "uv run isaaclab --editor", "problemMatcher": [] } ] diff --git a/tools/template/templates/external/.vscode/tools/settings.template.json b/tools/template/templates/external/.vscode/tools/settings.template.json index c1528d65dd73..d66c6f4f0322 100644 --- a/tools/template/templates/external/.vscode/tools/settings.template.json +++ b/tools/template/templates/external/.vscode/tools/settings.template.json @@ -72,8 +72,5 @@ }, "[restructuredtext]": { "editor.tabSize": 2 - }, - // Python extra paths - // Note: this is filled up when vscode is set up for the first time - "python.analysis.extraPaths": [] + } } diff --git a/tools/template/templates/external/.vscode/tools/setup_vscode.py b/tools/template/templates/external/.vscode/tools/setup_vscode.py deleted file mode 100644 index 9fcd115748ec..000000000000 --- a/tools/template/templates/external/.vscode/tools/setup_vscode.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""This script sets up the vs-code settings for the Isaac Lab project. - -This script merges the python.analysis.extraPaths from the "{ISAACSIM_DIR}/.vscode/settings.json" file into -the ".vscode/settings.json" file. - -This is necessary because Isaac Sim 2022.2.1 onwards does not add the necessary python packages to the python path -when the "setup_python_env.sh" is run as part of the vs-code launch configuration. -""" - -import os -import pathlib -import re -import subprocess -import sys - -ISAACLAB_DIR = pathlib.Path(__file__).parents[2] -"""Path to the Isaac Lab directory.""" - -# Try to find IsaacSim dir -_isaacsim_probe = subprocess.run( - [sys.executable, "-c", "import isaacsim; import os; print(os.environ.get('ISAAC_PATH', ''))"], - capture_output=True, - text=True, - check=False, - # avoid EULA prompt - stdin=subprocess.DEVNULL, -) -if _isaacsim_probe.returncode == 0 and _isaacsim_probe.stdout.strip(): - isaacsim_dir = _isaacsim_probe.stdout.strip() -else: - isaacsim_dir = os.path.join(ISAACLAB_DIR, "_isaac_sim") - -# check if the isaac-sim directory exists -if not os.path.exists(isaacsim_dir): - print( - f"[WARN] Could not find the isaac-sim directory: {isaacsim_dir}." - "\n\tIsaac Sim does not appear to be installed. VS Code settings will be generated" - "\n\twithout Isaac Sim extra paths." - ) - isaacsim_dir = "" - -ISAACSIM_DIR = isaacsim_dir -"""Path to the isaac-sim directory.""" - - -def overwrite_python_analysis_extra_paths(isaaclab_settings: str) -> str: - """Overwrite the python.analysis.extraPaths in the Isaac Lab settings file. - - The extraPaths are replaced with the path names from the isaac-sim settings file that exists in the - "{ISAACSIM_DIR}/.vscode/settings.json" file. - - If the isaac-sim settings file does not exist, the extraPaths are not overwritten. - - Args: - isaaclab_settings: The settings string to use as template. - - Returns: - The settings string with overwritten python analysis extra paths. - """ - # isaac-sim settings - isaacsim_vscode_filename = os.path.join(ISAACSIM_DIR, ".vscode", "settings.json") - - # we use the isaac-sim settings file to get the python.analysis.extraPaths for kit extensions - # if this file does not exist, we will not add any extra paths - if ISAACSIM_DIR and os.path.exists(isaacsim_vscode_filename): - # read the path names from the isaac-sim settings file - with open(isaacsim_vscode_filename) as f: - vscode_settings = f.read() - # extract the path names - # search for the python.analysis.extraPaths section and extract the contents - settings = re.search( - r"\"python.analysis.extraPaths\": \[.*?\]", vscode_settings, flags=re.MULTILINE | re.DOTALL - ) - settings = settings.group(0) - settings = settings.split('"python.analysis.extraPaths": [')[-1] - settings = settings.split("]")[0] - - # read the path names from the isaac-sim settings file - path_names = settings.split(",") - path_names = [path_name.strip().strip('"') for path_name in path_names] - path_names = [path_name for path_name in path_names if len(path_name) > 0] - - # change the path names to be relative to the Isaac Lab directory - rel_path = os.path.relpath(ISAACSIM_DIR, ISAACLAB_DIR) - path_names = ['"${workspaceFolder}/' + rel_path + "/" + path_name + '"' for path_name in path_names] - else: - path_names = [] - - # add the generated project's source directory - path_names.append('"${workspaceFolder}/src"') - - # combine them into a single string - path_names = ",\n\t\t".expandtabs(4).join(path_names) - # deal with the path separator being different on Windows and Unix - path_names = path_names.replace("\\", "/") - - # replace the path names in the Isaac Lab settings file with the path names parsed - isaaclab_settings = re.sub( - r"\"python.analysis.extraPaths\": \[.*?\]", - '"python.analysis.extraPaths": [\n\t\t'.expandtabs(4) + path_names + "\n\t]".expandtabs(4), - isaaclab_settings, - flags=re.DOTALL, - ) - # return the Isaac Lab settings string - return isaaclab_settings - - -def overwrite_default_python_interpreter(isaaclab_settings: str) -> str: - """Overwrite the default python interpreter in the Isaac Lab settings file. - - The default python interpreter is replaced with the path to the python interpreter used by the - isaac-sim project. This is necessary because the default python interpreter is the one shipped with - isaac-sim. - - Args: - isaaclab_settings: The settings string to use as template. - - Returns: - The settings string with overwritten default python interpreter. - """ - # read executable name - python_exe = sys.executable.replace("\\", "/") - - # We make an exception for replacing the default interpreter if the - # path (/kit/python/bin/python3) indicates that we are using a local/container - # installation of IsaacSim. We will preserve the calling script as the default, python.sh. - # We want to use python.sh because it modifies LD_LIBRARY_PATH and PYTHONPATH - # (among other envars) that we need for all of our dependencies to be accessible. - if "kit/python/bin/python3" in python_exe: - return isaaclab_settings - # replace the default python interpreter in the Isaac Lab settings file with the path to the - # python interpreter in the Isaac Lab directory - isaaclab_settings = re.sub( - r"\"python.defaultInterpreterPath\": \".*?\"", - f'"python.defaultInterpreterPath": "{python_exe}"', - isaaclab_settings, - flags=re.DOTALL, - ) - # return the Isaac Lab settings file - return isaaclab_settings - - -def main(): - # Isaac Lab template settings - isaaclab_vscode_template_filename = os.path.join(ISAACLAB_DIR, ".vscode", "tools", "settings.template.json") - # make sure the Isaac Lab template settings file exists - if not os.path.exists(isaaclab_vscode_template_filename): - raise FileNotFoundError( - f"Could not find the Isaac Lab template settings file: {isaaclab_vscode_template_filename}" - ) - # read the Isaac Lab template settings file - with open(isaaclab_vscode_template_filename) as f: - isaaclab_template_settings = f.read() - - # overwrite the python.analysis.extraPaths in the Isaac Lab settings file with the path names - isaaclab_settings = overwrite_python_analysis_extra_paths(isaaclab_template_settings) - # overwrite the default python interpreter in the Isaac Lab settings file with the path to the - # python interpreter used to call this script - isaaclab_settings = overwrite_default_python_interpreter(isaaclab_settings) - - # add template notice to the top of the file - header_message = ( - "// This file is a template and is automatically generated by the setup_vscode.py script.\n" - "// Do not edit this file directly.\n" - "// \n" - f"// Generated from: {isaaclab_vscode_template_filename}\n" - ) - isaaclab_settings = header_message + isaaclab_settings - - # write the Isaac Lab settings file - isaaclab_vscode_filename = os.path.join(ISAACLAB_DIR, ".vscode", "settings.json") - with open(isaaclab_vscode_filename, "w") as f: - f.write(isaaclab_settings) - - # copy the launch.json file if it doesn't exist - isaaclab_vscode_launch_filename = os.path.join(ISAACLAB_DIR, ".vscode", "launch.json") - isaaclab_vscode_template_launch_filename = os.path.join(ISAACLAB_DIR, ".vscode", "tools", "launch.template.json") - if not os.path.exists(isaaclab_vscode_launch_filename): - # read template launch settings - with open(isaaclab_vscode_template_launch_filename) as f: - isaaclab_template_launch_settings = f.read() - # add header - header_message = header_message.replace( - isaaclab_vscode_template_filename, isaaclab_vscode_template_launch_filename - ) - isaaclab_launch_settings = header_message + isaaclab_template_launch_settings - # write the Isaac Lab launch settings file - with open(isaaclab_vscode_launch_filename, "w") as f: - f.write(isaaclab_launch_settings) - - -if __name__ == "__main__": - main() diff --git a/tools/template/templates/external/README.md b/tools/template/templates/external/README.md index 01d0c188b59b..dd75f8fb616b 100644 --- a/tools/template/templates/external/README.md +++ b/tools/template/templates/external/README.md @@ -75,10 +75,36 @@ The test helpers under `source/isaaclab_tasks/test` in the Isaac Lab repository `isaaclab_tasks` package. Keep test fixtures in this project and use public Isaac Lab APIs. If you copy `env_test_utils.py`, it becomes vendored code whose upstream changes you must track. -To configure VS Code, run the `setup_python_env` task or invoke its command directly: +To configure VS Code or Cursor, run the `setup_python_env` task or invoke its command directly: ```bash -uv run python .vscode/tools/setup_vscode.py +uv run isaaclab --editor +``` + +The setup command selects the active interpreter and generates a git-ignored `pyrightconfig.json`. The generated +configuration inherits the project's checked-in Pyright settings and adds the Isaac Sim extensions, project `src` root, +and any Isaac Lab packages discovered in the active Python environment. This supports both Pylance in VS Code and +basedpyright in Cursor. + +In VS Code, use Pylance and select the interpreter that ran the setup command. In Cursor, install the +[basedpyright extension](https://marketplace.visualstudio.com/items?itemName=detachhead.basedpyright) instead of +Pylance, select the same interpreter, and reload the window. Both language servers read `pyrightconfig.json`. + +When using the `isaacsim` extra, include it while generating the editor configuration so the command can discover the +Isaac Sim installation: + +```bash +uv run --extra isaacsim isaaclab --editor +``` + +For an Isaac Sim binaries installation that is not available in the project environment, provide its path explicitly: + +```bash +# Linux +uv run isaaclab --editor --isaac_path + +# Windows +uv run isaaclab --editor --isaac_path ``` {% if include_ui_extension %} @@ -91,5 +117,6 @@ Add the project root to the Isaac Sim Extension Manager search paths, refresh, a {% endif %} ## Troubleshooting -If Pylance cannot resolve simulator modules, run the VS Code setup command above and reload the window. If indexing uses -too much memory, remove unused simulator extension paths from `.vscode/settings.json`. +If Pylance or basedpyright cannot resolve modules, confirm that the selected interpreter matches the one used to run the +setup command, then reload the editor window. To add a missing extension or reduce indexing memory, edit the `extraPaths` +array in the root `pyrightconfig.json`; remove simulator extension directories that the project does not use. diff --git a/tools/template/templates/external/pyproject.toml b/tools/template/templates/external/pyproject.toml index ae1b0ebb9d74..ff5a5acaa150 100644 --- a/tools/template/templates/external/pyproject.toml +++ b/tools/template/templates/external/pyproject.toml @@ -73,6 +73,11 @@ isaaclab = ["isaaclab", "isaaclab_newton", "isaaclab_ov", "isaaclab_physx"] [tool.ruff.format] docstring-code-format = true +[tool.pyright] +include = ["src", "scripts", "tests"] +exclude = ["**/__pycache__", "**/logs", ".git", ".venv", ".vscode"] +typeCheckingMode = "basic" + [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-ra" From cd48f96127471e107aa5253bafcc8a6ff9df8ef6 Mon Sep 17 00:00:00 2001 From: Neel Jawale Date: Tue, 8 Sep 2026 16:41:04 -0700 Subject: [PATCH 021/128] cuRobo install documentation change with conda GCC 14 toolchain (#7649) # Description `conda install -c nvidia cuda-toolkit=12.8` now pulls in a GCC 14 toolchain (the defaults channel jumped from GCC 11 to 14). This documentation change adds a small command fix. ## Type of change - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaacsim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- docs/source/overview/imitation-learning/skillgen.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/overview/imitation-learning/skillgen.rst b/docs/source/overview/imitation-learning/skillgen.rst index b471ea90e930..52b494be88b5 100644 --- a/docs/source/overview/imitation-learning/skillgen.rst +++ b/docs/source/overview/imitation-learning/skillgen.rst @@ -54,11 +54,14 @@ cuRobo provides the motion planning capabilities for SkillGen. This installation export PATH="$CUDA_HOME/bin:$PATH" && \ export LD_LIBRARY_PATH="$CUDA_HOME/lib:$LD_LIBRARY_PATH" && \ export TORCH_CUDA_ARCH_LIST="8.0+PTX" && \ + export CC=/usr/bin/gcc CXX=/usr/bin/g++ CUDAHOSTCXX=/usr/bin/g++ && \ pip install -e "git+https://github.com/NVlabs/curobo.git@ebb71702f3f70e767f40fd8e050674af0288abe8#egg=nvidia-curobo" --no-build-isolation .. note:: * The commit hash ``ebb71702f3f70e767f40fd8e050674af0288abe8`` is tested with Isaac Lab - using other versions may cause compatibility issues. This commit has the support for quad face mesh triangulation, required for cuRobo to parse usds as collision objects. + * The ``CC``/``CXX``/``CUDAHOSTCXX`` exports force the build to use the system compiler. Installing ``cuda-toolkit`` through Conda also pulls a Conda GCC toolchain (currently GCC 14) into the environment, which CUDA 12.8's ``nvcc`` does not support (it requires GCC < 14). The default system GCC on supported Ubuntu versions (GCC 11 on 22.04, GCC 13 on 24.04) is compatible. + * cuRobo is installed from source and is editable installed. This means that the cuRobo source code will be cloned in the current directory under ``src/nvidia-curobo``. Users can choose their working directory to install cuRobo. * ``TORCH_CUDA_ARCH_LIST`` in the above command should match your GPU's CUDA compute capability (e.g., ``8.0`` for A100, ``8.6`` for many RTX 30‑series, ``8.9`` for RTX 4090); the ``+PTX`` suffix embeds PTX for forward compatibility so newer GPUs can JIT‑compile when native SASS isn’t included. From 59d2807361e3dbd7340df7537cd802f4fed61e13 Mon Sep 17 00:00:00 2001 From: Mustafa H <34825877+StafaH@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:44:32 -0400 Subject: [PATCH 022/128] [Workflow] Fix template generator setup workflows (#7645) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Fix three failures in the documented template-generator workflows: - Pin generated external projects and their optional extras to the exact Isaac Lab version that ran the generator. This prevents `uv sync` from silently selecting an older prerelease whose CLI may do nothing and return success. - Let the repository environment-listing script run without constructing `AppLauncher`, so task and preset discovery works in the default Kit-less environment. - Document the explicit non-PPO agent entry point for generated internal tasks while retaining automatic selection when the task exposes one unambiguous agent configuration. No new dependencies are required. Internal bug: NVBug 6695420 validation follow-up. ## Type of change - Bug fix (non-breaking change which fixes an issue) - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable. ## Validation - `uv run --frozen python -m pytest --confcutdir=tools/template tools/template/test_cli.py -q` — 7 passed - `uv run --frozen python -m pytest --confcutdir=tools/test tools/test/test_list_envs.py -q` — 1 passed - `uv run --frozen python -m pytest --confcutdir=source/isaaclab_tasks source/isaaclab_tasks/test/core/test_preset_cli.py -q` — 31 passed - `uv run --frozen python -m pytest --confcutdir=source/isaaclab source/isaaclab/test/cli/test_installed_workflow_entrypoints.py -q` — 10 passed - `uv run --isolated --extra dev -- make -C docs current-docs` — passed without warnings - `uv run --frozen isaaclab -f` — passed ## Checklist - [x] I have read and understood the contribution guidelines - [x] I have run the pre-commit checks - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added focused tests that prove the fixes are effective - [x] No package changelog fragment is required because no `source//` code changed - [x] My name already exists in `CONTRIBUTORS.md` --- .../developer-tools/template_generator.rst | 13 ++++++++ scripts/environments/list_envs.py | 20 +------------ tools/template/cli.py | 1 + .../templates/external/pyproject.toml | 10 +++---- tools/template/test_cli.py | 14 +++++---- tools/test/test_list_envs.py | 30 +++++++++++++++++++ 6 files changed, 58 insertions(+), 30 deletions(-) create mode 100644 tools/test/test_list_envs.py diff --git a/docs/source/developer-tools/template_generator.rst b/docs/source/developer-tools/template_generator.rst index 4d426df9d49d..410575bf6997 100644 --- a/docs/source/developer-tools/template_generator.rst +++ b/docs/source/developer-tools/template_generator.rst @@ -70,6 +70,9 @@ installed Isaac Lab package: The command uses the dependencies from the active Isaac Lab environment. It does not invoke ``pip`` or install another set of template dependencies, so it also works in the pip-less virtual environments created by ``uv``. +The generated ``pyproject.toml`` pins Isaac Lab and its optional extras to that +environment's exact Isaac Lab version so ``uv sync`` cannot silently resolve an +older release. The short form is equivalent: @@ -277,6 +280,16 @@ a separate project. From the Isaac Lab repository root, list and test it with: uv run isaaclab random_agent --task --num_envs 16 uv run isaaclab train --rl_library --task +The training command automatically selects an agent configuration when the +generated task has only one entry point for that RL library. If you generated +multiple algorithms, or want to select one explicitly, pass its registered +entry-point name. Non-PPO entry points include the algorithm name; for example: + +.. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl --task \ + --agent rsl_rl_distillation_cfg_entry_point + Troubleshooting --------------- diff --git a/scripts/environments/list_envs.py b/scripts/environments/list_envs.py index 2d5b2d0a0d01..5b376cf979a9 100644 --- a/scripts/environments/list_envs.py +++ b/scripts/environments/list_envs.py @@ -13,13 +13,9 @@ with `Isaac` in their name. """ -"""Launch Isaac Sim Simulator first.""" - import argparse import contextlib -from isaaclab.app import AppLauncher - # add argparse arguments parser = argparse.ArgumentParser(description="List Isaac Lab environments.") parser.add_argument("--keyword", type=str, default=None, help="Keyword to filter environments.") @@ -36,13 +32,6 @@ # parse the arguments args_cli = parser.parse_args() -# launch omniverse app -app_launcher = AppLauncher(headless=True) -simulation_app = app_launcher.app - - -"""Rest everything follows.""" - import gymnasium as gym from prettytable import PrettyTable @@ -127,11 +116,4 @@ def main(): if __name__ == "__main__": - try: - # run the main function - main() - except Exception as e: - raise e - finally: - # close the app - simulation_app.close() + main() diff --git a/tools/template/cli.py b/tools/template/cli.py index 770a5fd10b78..03deb3e8406f 100644 --- a/tools/template/cli.py +++ b/tools/template/cli.py @@ -296,6 +296,7 @@ def main() -> None: "external": is_external_project, "path": project_path, "name": project_name, + "isaaclab_version": lab_module.__version__, "task_name": task_name, "robot_name": robot_name, "include_ui_extension": include_ui_extension, diff --git a/tools/template/templates/external/pyproject.toml b/tools/template/templates/external/pyproject.toml index ff5a5acaa150..3c6262bd00fd 100644 --- a/tools/template/templates/external/pyproject.toml +++ b/tools/template/templates/external/pyproject.toml @@ -16,17 +16,17 @@ license = { file = "LICENSE" } authors = [{ name = "Isaac Lab Project Developers" }] requires-python = ">=3.12,<3.13" dependencies = [ - "isaaclab{% if rl_libraries %}[{% for rl_library in rl_libraries %}{% if not loop.first %},{% endif %}{{ rl_library.name | replace('_', '-') }}{% endfor %}]{% endif %}", + "isaaclab{% if rl_libraries %}[{% for rl_library in rl_libraries %}{% if not loop.first %},{% endif %}{{ rl_library.name | replace('_', '-') }}{% endfor %}]{% endif %}{% if isaaclab_version %}=={{ isaaclab_version }}{% endif %}", ] [project.entry-points."isaaclab.tasks"] "{{ name }}" = "{{ name }}.tasks" [project.optional-dependencies] -isaacsim = ["isaaclab[isaacsim]"] -ov = ["isaaclab[ov]"] -ovphysx = ["isaaclab[ovphysx]"] -ovrtx = ["isaaclab[ovrtx]"] +isaacsim = ["isaaclab[isaacsim]{% if isaaclab_version %}=={{ isaaclab_version }}{% endif %}"] +ov = ["isaaclab[ov]{% if isaaclab_version %}=={{ isaaclab_version }}{% endif %}"] +ovphysx = ["isaaclab[ovphysx]{% if isaaclab_version %}=={{ isaaclab_version }}{% endif %}"] +ovrtx = ["isaaclab[ovrtx]{% if isaaclab_version %}=={{ isaaclab_version }}{% endif %}"] [dependency-groups] dev = ["codespell>=2.4", "pre-commit>=4.2", "pytest>=8.3", "ruff>=0.11"] diff --git a/tools/template/test_cli.py b/tools/template/test_cli.py index af8889a223e1..3b62759b3fb4 100644 --- a/tools/template/test_cli.py +++ b/tools/template/test_cli.py @@ -42,6 +42,7 @@ def _external_specification(tmp_path: Path, include_ui_extension: bool = False) "external": True, "path": str(tmp_path), "name": "test_project", + "isaaclab_version": "3.0.0", "task_name": "place_vial", "robot_name": "so101", "include_ui_extension": include_ui_extension, @@ -106,7 +107,7 @@ def test_main_collects_canonical_external_project_choices(): handler.input_checkbox.side_effect = lambda message, choices: [choices[0]] handler.get_choices.side_effect = CLIHandler.get_choices - source_install = types.SimpleNamespace(__file__="/repo/source/isaaclab/isaaclab/__init__.py") + source_install = types.SimpleNamespace(__file__="/repo/source/isaaclab/isaaclab/__init__.py", __version__="3.0.0") with ( mock.patch.object(_MODULE, "CLIHandler", return_value=handler), mock.patch.object(_MODULE.importlib, "import_module", return_value=source_install), @@ -125,6 +126,7 @@ def test_main_collects_canonical_external_project_choices(): assert specification["task_name"] == "place_vial" assert specification["robot_name"] == "so101" assert specification["include_ui_extension"] is False + assert specification["isaaclab_version"] == "3.0.0" def test_generated_project_matches_canonical_uv_layout(tmp_path): @@ -142,13 +144,13 @@ def test_generated_project_matches_canonical_uv_layout(tmp_path): "requires": ["uv_build>=0.12.6,<0.13"], "build-backend": "uv_build", } - assert project_config["project"]["dependencies"] == ["isaaclab[rsl-rl]"] + assert project_config["project"]["dependencies"] == ["isaaclab[rsl-rl]==3.0.0"] assert project_config["project"]["entry-points"]["isaaclab.tasks"] == {"test_project": "test_project.tasks"} assert project_config["project"]["optional-dependencies"] == { - "isaacsim": ["isaaclab[isaacsim]"], - "ov": ["isaaclab[ov]"], - "ovphysx": ["isaaclab[ovphysx]"], - "ovrtx": ["isaaclab[ovrtx]"], + "isaacsim": ["isaaclab[isaacsim]==3.0.0"], + "ov": ["isaaclab[ov]==3.0.0"], + "ovphysx": ["isaaclab[ovphysx]==3.0.0"], + "ovrtx": ["isaaclab[ovrtx]==3.0.0"], } assert project_config["dependency-groups"]["dev"] == [ "codespell>=2.4", diff --git a/tools/test/test_list_envs.py b/tools/test/test_list_envs.py new file mode 100644 index 000000000000..9c4515ca891d --- /dev/null +++ b/tools/test/test_list_envs.py @@ -0,0 +1,30 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the environment-listing script.""" + +import subprocess +import sys +from pathlib import Path + + +def test_list_envs_runs_without_launching_isaac_sim() -> None: + """Environment discovery must work in the default Kit-less environment.""" + repository_root = Path(__file__).parents[2] + result = subprocess.run( + [ + sys.executable, + str(repository_root / "scripts" / "environments" / "list_envs.py"), + "--keyword", + "Nonexistent-Task", + "--show_presets", + ], + cwd=repository_root, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "Available Environments in Isaac Lab" in result.stdout From 6fdc484cf757531a13a298293ecd43bc00e002c1 Mon Sep 17 00:00:00 2001 From: shauryadNv Date: Tue, 8 Sep 2026 16:50:21 -0700 Subject: [PATCH 023/128] Bug Fixes for Mimic-Cosmos Workflows (#7500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description This PR bundles four fixes for issues found: 1. 6684416: **RTX renderer: fix crash on not-yet-warmed-up annotator buffers.** `IsaacRtxRenderer.render()` unconditionally trimmed the tiled buffer to 2/3 channels for `motion_vectors`/`normals`/`SIMPLE_SHADING_MODES`/`RGB_HDR`. Immediately after an annotator is attached (e.g. at env creation, before the RTX renderer has pumped a frame), Replicator can momentarily return a buffer whose channel dimension is 0. Warp's array slicing rejects trimming an already-empty dimension (unlike NumPy, which allows it), raising `RuntimeError: Invalid indexing in slice: 0:0:1`. Guard the trim and skip writing that data type for the frame instead of crashing. 2. 6683697: **`robust_eval.py`/`play.py`: pass `enable_cameras=True` explicitly.** `release/3.0.0` removed the `--enable_cameras` CLI flag and the `ENABLE_CAMERAS` env-var fallback from `AppLauncher` in favor of `launch_simulation()`'s scene-scan auto-detection, but these two robomimic scripts still use the legacy `AppLauncher(args_cli)` pattern and were never migrated. Without cameras enabled, and with the old explicit "pass --enable_cameras" guard in `IsaacRtxRenderer.__init__` also removed in this release, the failure now surfaces as an opaque `ValueError: Invalid object in Py_Graph in getWrappedGraphFromNode` deep inside OmniGraph/SyntheticData. Pass `enable_cameras=True` explicitly, matching the pattern already used in `generate_dataset.py`. 3. 6683610: **Docs: add `uv run` examples for the HDF5/MP4 conversion and merge tools.** The Augmented Imitation Learning doc recommends `uv` as the primary workflow throughout (dataset generation, training, eval all show a "uv (Recommended)" tab), but the `hdf5_to_mp4.py`, `mp4_to_hdf5.py`, and `merge_hdf5_datasets.py` examples only showed a bare `python ...` invocation with no indication of which environment/extras to use. Added matching `uv (Recommended)` / `isaaclab.sh` tab-sets (`--extra mimic`, since these tools only need `h5py`/`opencv`/`numpy`, no Isaac Sim import). 4. 6702683: **RTX renderer: fail fast on oversized tiled camera buffers.** `IsaacRtxRenderer.render()` flattens every environment's camera tile into one Warp array per data type. Warp requires every array dimension to fit in a signed 32-bit int, so a large enough `num_envs * resolution` combination overflows that limit and crashes deep inside `render()`'s `.flatten()` call with `ValueError: Array shapes must not exceed the maximum representable value of a signed 32-bit integer, got 2621440000 in dimension 0` and no indication of what to change. Compute the worst-case tiled buffer size upfront in `create_render_data()` and raise a clear error naming the offending env count/grid/resolution and suggesting to reduce `--num_envs` or camera resolution. This does not lift the underlying Warp/RTX limit — it only replaces the opaque failure with an actionable one. ## Type of change - Bug fix (non-breaking change which fixes an issue) - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] I have added a changelog fragment under `source//changelog.d/` for every touched package (do **not** edit `CHANGELOG.rst` or bump `extension.toml` — CI handles that) - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- .../augmented_imitation.rst | 84 ++++++++++++++++--- scripts/imitation_learning/robomimic/play.py | 12 ++- .../robomimic/robust_eval.py | 12 ++- .../changelog.d/shauryad-bug-fixes.rst | 7 ++ .../isaaclab/isaaclab/utils/warp/kernels.py | 25 +++--- .../changelog.d/shauryad-bug-fixes.rst | 6 ++ .../renderers/isaac_rtx_renderer.py | 23 ++++- 7 files changed, 139 insertions(+), 30 deletions(-) create mode 100644 source/isaaclab/changelog.d/shauryad-bug-fixes.rst create mode 100644 source/isaaclab_physx/changelog.d/shauryad-bug-fixes.rst diff --git a/docs/source/overview/imitation-learning/augmented_imitation.rst b/docs/source/overview/imitation-learning/augmented_imitation.rst index 296ce1426094..082f499da165 100644 --- a/docs/source/overview/imitation-learning/augmented_imitation.rst +++ b/docs/source/overview/imitation-learning/augmented_imitation.rst @@ -54,6 +54,27 @@ requires. Cosmos Augmentation ~~~~~~~~~~~~~~~~~~~ +.. important:: + The ``hdf5_to_mp4.py`` and ``mp4_to_hdf5.py`` scripts below read and write MP4 files through OpenCV's + video I/O, which requires ffmpeg support. Recent Isaac Sim releases ship an OpenCV build without ffmpeg, + so install the full OpenCV package into the environment before running these conversions: + + .. tab-set:: + + .. tab-item:: uv (Recommended) + + .. code:: bash + + uv pip install opencv-python + + .. tab-item:: isaaclab.sh / isaaclab.bat + + .. code:: bash + + ./isaaclab.sh -p -m pip install opencv-python + + Without it, the conversion scripts fail to open or write the video files. + HDF5 to MP4 Conversion ^^^^^^^^^^^^^^^^^^^^^^ @@ -93,11 +114,23 @@ The ``hdf5_to_mp4.py`` script converts camera frames stored in HDF5 demonstratio Example usage for the cube stacking task: -.. code:: bash +.. tab-set:: + + .. tab-item:: uv (Recommended) - python scripts/tools/hdf5_to_mp4.py \ - --input_file datasets/mimic_dataset_1k.hdf5 \ - --output_dir datasets/mimic_dataset_1k_mp4 + .. code:: bash + + uv run --extra mimic python scripts/tools/hdf5_to_mp4.py \ + --input_file datasets/mimic_dataset_1k.hdf5 \ + --output_dir datasets/mimic_dataset_1k_mp4 + + .. tab-item:: isaaclab.sh / isaaclab.bat + + .. code:: bash + + ./isaaclab.sh -p scripts/tools/hdf5_to_mp4.py \ + --input_file datasets/mimic_dataset_1k.hdf5 \ + --output_dir datasets/mimic_dataset_1k_mp4 .. _running-cosmos: @@ -251,12 +284,25 @@ The ``mp4_to_hdf5.py`` script converts the visually augmented MP4 videos back to Example usage for the cube stacking task: -.. code:: bash +.. tab-set:: + + .. tab-item:: uv (Recommended) - python scripts/tools/mp4_to_hdf5.py \ - --input_file datasets/mimic_dataset_1k.hdf5 \ - --videos_dir datasets/cosmos_dataset_1k_mp4 \ - --output_file datasets/cosmos_dataset_1k.hdf5 + .. code:: bash + + uv run --extra mimic python scripts/tools/mp4_to_hdf5.py \ + --input_file datasets/mimic_dataset_1k.hdf5 \ + --videos_dir datasets/cosmos_dataset_1k_mp4 \ + --output_file datasets/cosmos_dataset_1k.hdf5 + + .. tab-item:: isaaclab.sh / isaaclab.bat + + .. code:: bash + + ./isaaclab.sh -p scripts/tools/mp4_to_hdf5.py \ + --input_file datasets/mimic_dataset_1k.hdf5 \ + --videos_dir datasets/cosmos_dataset_1k_mp4 \ + --output_file datasets/cosmos_dataset_1k.hdf5 Pre-generated Dataset ^^^^^^^^^^^^^^^^^^^^^ @@ -291,11 +337,23 @@ The ``merge_hdf5_datasets.py`` script combines multiple HDF5 datasets into a sin Example usage for the cube stacking task: -.. code:: bash +.. tab-set:: + + .. tab-item:: uv (Recommended) + + .. code:: bash + + uv run --extra mimic python scripts/tools/merge_hdf5_datasets.py \ + --input_files datasets/mimic_dataset_1k.hdf5 datasets/cosmos_dataset_1k.hdf5 \ + --output_file datasets/mimic_cosmos_dataset.hdf5 + + .. tab-item:: isaaclab.sh / isaaclab.bat + + .. code:: bash - python scripts/tools/merge_hdf5_datasets.py \ - --input_files datasets/mimic_dataset_1k.hdf5 datasets/cosmos_dataset_1k.hdf5 \ - --output_file datasets/mimic_cosmos_dataset.hdf5 + ./isaaclab.sh -p scripts/tools/merge_hdf5_datasets.py \ + --input_files datasets/mimic_dataset_1k.hdf5 datasets/cosmos_dataset_1k.hdf5 \ + --output_file datasets/mimic_cosmos_dataset.hdf5 Model Training and Evaluation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scripts/imitation_learning/robomimic/play.py b/scripts/imitation_learning/robomimic/play.py index c07a842def7e..8455e9e3ff72 100644 --- a/scripts/imitation_learning/robomimic/play.py +++ b/scripts/imitation_learning/robomimic/play.py @@ -22,7 +22,9 @@ import argparse -from isaaclab.app import AppLauncher +from isaaclab.app import AppLauncher, scan + +from isaaclab_tasks.utils import resolve_task_config # add argparse arguments parser = argparse.ArgumentParser(description="Evaluate robomimic policy for Isaac Lab environment.") @@ -47,7 +49,13 @@ args_cli, hydra_overrides = parser.parse_known_args() # launch omniverse app -app_launcher = AppLauncher(args_cli) +# Only enable rendering for tasks that actually declare Kit camera sensors: this script also +# plays policies trained on low-dimensional observations, which should not pay for the RTX +# renderer. ``resolve_task_config`` is safe to call before Kit is launched, and ``scan`` is the +# same detection ``launch_simulation`` uses, so this matches how camera enabling is resolved +# elsewhere now that the ``--enable_cameras`` flag is gone. +env_cfg_for_scan, _ = resolve_task_config(args_cli.task, "") +app_launcher = AppLauncher(args_cli, enable_cameras=scan(env_cfg_for_scan, args_cli).has_kit_camera) simulation_app = app_launcher.app """Rest everything follows.""" diff --git a/scripts/imitation_learning/robomimic/robust_eval.py b/scripts/imitation_learning/robomimic/robust_eval.py index df2c534704b4..0cd4ead6fd4e 100644 --- a/scripts/imitation_learning/robomimic/robust_eval.py +++ b/scripts/imitation_learning/robomimic/robust_eval.py @@ -28,7 +28,9 @@ import argparse -from isaaclab.app import AppLauncher +from isaaclab.app import AppLauncher, scan + +from isaaclab_tasks.utils import resolve_task_config # add argparse arguments parser = argparse.ArgumentParser(description="Evaluate robomimic policy for Isaac Lab environment.") @@ -64,7 +66,13 @@ args_cli, hydra_overrides = parser.parse_known_args() # launch omniverse app -app_launcher = AppLauncher(args_cli) +# Only enable rendering for tasks that actually declare Kit camera sensors: this script also +# evaluates policies trained on low-dimensional observations, which should not pay for the RTX +# renderer. ``resolve_task_config`` is safe to call before Kit is launched, and ``scan`` is the +# same detection ``launch_simulation`` uses, so this matches how camera enabling is resolved +# elsewhere now that the ``--enable_cameras`` flag is gone. +env_cfg_for_scan, _ = resolve_task_config(args_cli.task, "") +app_launcher = AppLauncher(args_cli, enable_cameras=scan(env_cfg_for_scan, args_cli).has_kit_camera) simulation_app = app_launcher.app """Rest everything follows.""" diff --git a/source/isaaclab/changelog.d/shauryad-bug-fixes.rst b/source/isaaclab/changelog.d/shauryad-bug-fixes.rst new file mode 100644 index 000000000000..ab2ea18cc3a1 --- /dev/null +++ b/source/isaaclab/changelog.d/shauryad-bug-fixes.rst @@ -0,0 +1,7 @@ +Changed +^^^^^^^ + +* Changed the ``reshape_tiled_image`` Warp kernel to index the tiled image buffer as a 3D array + of shape (num_tiles_y * image_height, num_tiles_x * image_width, num_channels) instead of a + flattened 1D array. This keeps every array dimension within Warp's per-dimension size limit, so + large environment counts and camera resolutions no longer overflow a single flattened dimension. diff --git a/source/isaaclab/isaaclab/utils/warp/kernels.py b/source/isaaclab/isaaclab/utils/warp/kernels.py index 5a8e68264918..eda3973e5dc3 100644 --- a/source/isaaclab/isaaclab/utils/warp/kernels.py +++ b/source/isaaclab/isaaclab/utils/warp/kernels.py @@ -344,8 +344,14 @@ def reshape_tiled_image( is assumed to be tiled in the x and y directions. The output image is a batch of images with the specified height, width, and number of channels. + The tiled buffer is indexed as a 3D array rather than flattened to 1D so that the number of + cameras and the camera resolution are bounded per dimension instead of by their product. A + flattened view of a large tiled buffer can exceed the maximum size of a single Warp array + dimension, see https://nvidia.github.io/warp/stable/user_guide/limitations.html#arrays. + Args: - tiled_image_buffer: The input image buffer. Shape is (height * width * num_channels * num_cameras,). + tiled_image_buffer: The input image buffer. Shape is + (num_tiles_y * image_height, num_tiles_x * image_width, num_channels). batched_image: The output image. Shape is (num_cameras, height, width, num_channels). image_width: The width of the image. image_height: The height of the image. @@ -358,32 +364,29 @@ def reshape_tiled_image( # resolve the tile indices tile_x_id = camera_id % num_tiles_x tile_y_id = camera_id // num_tiles_x - # compute the start index of the pixel in the tiled image buffer - pixel_start = ( - num_channels * num_tiles_x * image_width * (image_height * tile_y_id + height_id) - + num_channels * tile_x_id * image_width - + num_channels * width_id - ) + # resolve the pixel position within the tiled image buffer + row = image_height * tile_y_id + height_id + col = image_width * tile_x_id + width_id # copy the pixel values into the batched image for i in range(num_channels): - batched_image[camera_id, height_id, width_id, i] = batched_image.dtype(tiled_image_buffer[pixel_start + i]) + batched_image[camera_id, height_id, width_id, i] = batched_image.dtype(tiled_image_buffer[row, col, i]) # uint32 -> int32 conversion is required for non-colored segmentation annotators wp.overload( reshape_tiled_image, - {"tiled_image_buffer": wp.array(dtype=wp.uint32), "batched_image": wp.array(dtype=wp.uint32, ndim=4)}, + {"tiled_image_buffer": wp.array(dtype=wp.uint32, ndim=3), "batched_image": wp.array(dtype=wp.uint32, ndim=4)}, ) # uint8 is used for 4 channel annotators wp.overload( reshape_tiled_image, - {"tiled_image_buffer": wp.array(dtype=wp.uint8), "batched_image": wp.array(dtype=wp.uint8, ndim=4)}, + {"tiled_image_buffer": wp.array(dtype=wp.uint8, ndim=3), "batched_image": wp.array(dtype=wp.uint8, ndim=4)}, ) # float32 is used for single channel annotators wp.overload( reshape_tiled_image, - {"tiled_image_buffer": wp.array(dtype=wp.float32), "batched_image": wp.array(dtype=wp.float32, ndim=4)}, + {"tiled_image_buffer": wp.array(dtype=wp.float32, ndim=3), "batched_image": wp.array(dtype=wp.float32, ndim=4)}, ) ## diff --git a/source/isaaclab_physx/changelog.d/shauryad-bug-fixes.rst b/source/isaaclab_physx/changelog.d/shauryad-bug-fixes.rst new file mode 100644 index 000000000000..22ece077d3b0 --- /dev/null +++ b/source/isaaclab_physx/changelog.d/shauryad-bug-fixes.rst @@ -0,0 +1,6 @@ +Changed +^^^^^^^ + +* Changed ``IsaacRtxRenderer.render()`` to pass the tiled annotator buffer to + ``reshape_tiled_image`` as a 3D array instead of flattening it to 1D. Large environment counts + and camera resolutions no longer overflow the maximum size of a single Warp array dimension. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 9c984e9f0378..7a9122ebf058 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -573,7 +573,7 @@ def tiling_grid_shape(): rows = math.ceil(view_count / cols) return (cols, rows) - num_tiles_x = tiling_grid_shape()[0] + num_tiles_x, num_tiles_y = tiling_grid_shape() # Extract the flattened image buffer for data_type, annotator in render_data.annotators.items(): @@ -630,11 +630,30 @@ def tiling_grid_shape(): if data_type == str(RenderBufferKind.RGB_HDR): tiled_data_buffer = tiled_data_buffer[:, :, :3].contiguous() + # ``reshape_tiled_image`` indexes the tiled buffer as + # (num_tiles_y * height, num_tiles_x * width, channels), but annotators hand this data back + # with varying shapes: 3D for multi-channel outputs, 2D for single-channel ones, and — for the + # colorized segmentation types reinterpreted above — a descriptor that over-claims the backing + # memory when the raw buffer already carries a channel axis (e.g. (H, W, 4) becomes (H, W, 4, 4)). + # Build the view directly from the pointer rather than reshaping, so the extra claimed elements + # are ignored exactly as the previous flattened indexing ignored them. Keeping the view 3D + # instead of 1D also keeps every dimension within Warp's per-dimension array size limit, so + # large environment counts and camera resolutions no longer overflow a flattened dimension. + tile_height, tile_width, num_channels = (int(dim) for dim in buf_wp.shape[1:]) + # ``tiled_source`` must outlive the view below: the view does not own the annotator memory. + tiled_source = tiled_data_buffer + tiled_data_buffer = wp.array( + ptr=tiled_source.ptr, + shape=(num_tiles_y * tile_height, num_tiles_x * tile_width, num_channels), + dtype=tiled_source.dtype, + device=device, + ) + wp.launch( kernel=reshape_tiled_image, dim=(view_count, cfg.height, cfg.width), inputs=[ - tiled_data_buffer.flatten(), + tiled_data_buffer, buf_wp, *list(buf_wp.shape[1:]), num_tiles_x, From cdaefc505308d97b5fd2289d8c997e4f307dd8b4 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Tue, 8 Sep 2026 16:56:19 -0700 Subject: [PATCH 024/128] [Tasks] Support pretrained Cartpole feature checkpoints (#7630) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Complete preset-aware pretrained checkpoint support for feature-extractor policies on `Isaac-Cartpole-Camera`. - Advertise every feature-agent preset supported by the preferred RSL-RL workflow in the checkpoint publication matrix. The currently registered feature presets are `resnet18` and `theia_tiny`. - Flatten arbitrary multi-dimensional frozen image-encoder outputs after the environment batch dimension. This is a no-op for ResNet18's already-flat output and converts Theia-Tiny's token output from `(num_envs, 36, 192)` to the flat observation required by the MLP policy. - Validate checkpoint metadata registry-wide: workflows must be registered, presets must be unique, and every advertised preset must be a real domain preset. - Derive the Cartpole coverage assertion from `agent_preset_compatibility`, so adding a future feature preset without checkpoint publication metadata fails CI. - Regenerate the environment browser so supported pretrained feature presets are visible. The mechanism is not tied to the two QA commands: #7594 provides generic preset-aware naming/lookup/publication, while this PR covers arbitrary feature encoder shapes and future feature-preset registry additions. Other raw camera/rendering/domain presets remain explicit opt-ins because each advertised entry requires a compatible trained artifact. This builds on the agent preset selection fixed by #7532 and the checkpoint naming/lookup fixed by #7594. Both are merged. Addresses NVBug 6675381. ## Type of change - Bug fix (non-breaking change which fixes an issue) - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable. ## Validation - Regression test failed on merged `develop`: encoder output remained `torch.Size([4, 6, 8])` instead of `(4, 48)`. - `.venv/bin/python -m pytest scripts/tools/test/test_train_and_publish_checkpoints.py source/isaaclab/test/envs/test_stacked_image_mdp.py source/isaaclab_rl/test/test_pretrained_checkpoint.py -q` — 44 passed. - `UV_FROZEN=1 uv run isaaclab -f` — passed. - `uv run python tools/changelog/cli.py check develop` — passed. - `uv run --isolated --extra test -- make -C docs current-docs` — generated the updated browser and completed all pages, but exited 2 because the local isolated environment reports 44 pre-existing import/reference warnings as errors (including unavailable `ovstage` and `isaaclab_ppisp` modules). The PR docs CI is the authoritative warning-free build. - ResNet18 and Theia-Tiny RSL-RL policies trained for 1,638,400 steps each with local Isaac Sim 6.1 and reached mean episode lengths of 222.25 and 177.00, respectively (benchmark threshold: 150). - Both checkpoints loaded and reached policy playback; Theia-Tiny used the corrected flattened observation size of 6,912. - Both default Newton MJWarp/Newton renderer artifacts are present on the internal Nucleus server under the exact preset-aware filenames: - `rsl_rl/Isaac-Cartpole-Camera_resnet18_newtonmjwarp_newton_rsl_rl.pt` — 6,985,267 bytes, SHA-256 `13cbec819675809d6904c57d68a20d7c7d92d283999502509e8fd8da006099f8` - `rsl_rl/Isaac-Cartpole-Camera_theia_tiny_newtonmjwarp_newton_rsl_rl.pt` — 43,450,675 bytes, SHA-256 `9f75a2069bdec9bd06f888b9264da3ad664d3707fa038cf09929045ce5b526c0` ## Artifact coverage The uploaded pair covers the default Newton MJWarp/Newton renderer selection. The generic core publication matrix also schedules both feature presets for these additional supported backend/renderer pairs, whose artifacts still need to be trained and published for full cross-backend coverage: - PhysX / RTX - PhysX / Newton renderer - Newton MJWarp / RTX ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `uv run isaaclab -f` - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package - [x] My name already exists in `CONTRIBUTORS.md` --- .../source/_static/css/environment-browser.js | 2 +- .../test_train_and_publish_checkpoints.py | 30 +++++++++++++++++++ .../changelog.d/flatten-image-features.rst | 5 ++++ .../isaaclab/envs/mdp/observations.py | 4 +-- .../test/envs/test_stacked_image_mdp.py | 16 +++++++++- .../cartpole-feature-checkpoints.rst | 4 +++ .../isaaclab_tasks/core/cartpole/__init__.py | 6 ++-- 7 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 source/isaaclab/changelog.d/flatten-image-features.rst create mode 100644 source/isaaclab_tasks/changelog.d/cartpole-feature-checkpoints.rst diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index 86f9a745e306..54835689ec66 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -17,7 +17,7 @@ ["Isaac-Cartpole-Direct", "rl_games,rsl_rl,skrl,sb3", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/cartpole.jpg", true], ["Isaac-Cartpole", "rl_games,rsl_rl,skrl,sb3", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/cartpole.jpg", true], ["Isaac-Cartpole-Camera-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl", {}, "tasks/classic/cartpole.jpg", false, {"*": ["rgb"], "rl_games": ["depth"]}], - ["Isaac-Cartpole-Camera", "rl_games,rsl_rl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,resnet18,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl,theia_tiny", {"rl_games_cfg_entry_point": ["albedo", "depth", "rgb", "semantic_segmentation", "simple_shading_constant_diffuse", "simple_shading_diffuse_mdl", "simple_shading_full_mdl"], "rl_games_feature_cfg_entry_point": ["resnet18", "theia_tiny"], "rsl_rl_cfg_entry_point": ["albedo", "depth", "rgb", "semantic_segmentation", "simple_shading_constant_diffuse", "simple_shading_diffuse_mdl", "simple_shading_full_mdl"], "rsl_rl_feature_cfg_entry_point": ["resnet18", "theia_tiny"]}, "tasks/classic/cartpole.jpg", false, {"*": ["rgb"]}], + ["Isaac-Cartpole-Camera", "rl_games,rsl_rl", "isaacsim_physx,newton_kamino,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "albedo,depth,resnet18,rgb,semantic_segmentation,simple_shading_constant_diffuse,simple_shading_diffuse_mdl,simple_shading_full_mdl,theia_tiny", {"rl_games_cfg_entry_point": ["albedo", "depth", "rgb", "semantic_segmentation", "simple_shading_constant_diffuse", "simple_shading_diffuse_mdl", "simple_shading_full_mdl"], "rl_games_feature_cfg_entry_point": ["resnet18", "theia_tiny"], "rsl_rl_cfg_entry_point": ["albedo", "depth", "rgb", "semantic_segmentation", "simple_shading_constant_diffuse", "simple_shading_diffuse_mdl", "simple_shading_full_mdl"], "rsl_rl_feature_cfg_entry_point": ["resnet18", "theia_tiny"]}, "tasks/classic/cartpole.jpg", false, {"*": ["rgb"], "rsl_rl": ["resnet18", "theia_tiny"]}], ["Isaac-Fourbar-Pole-Swingup", "rsl_rl", "newton_kamino", "", "", {}, "tasks/classic/fourbar_pole.jpg"], ["Isaac-Humanoid-Direct", "rl_games,rsl_rl,skrl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/humanoid.jpg", true], ["Isaac-Humanoid", "rl_games,rsl_rl,skrl,sb3", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", {}, "tasks/classic/humanoid.jpg", true], diff --git a/scripts/tools/test/test_train_and_publish_checkpoints.py b/scripts/tools/test/test_train_and_publish_checkpoints.py index 01f59eb4fb6c..f7b555a9e341 100644 --- a/scripts/tools/test/test_train_and_publish_checkpoints.py +++ b/scripts/tools/test/test_train_and_publish_checkpoints.py @@ -9,8 +9,10 @@ from pathlib import Path from types import SimpleNamespace +import gymnasium as gym import pytest +from isaaclab_tasks.utils.preset_cli import enumerate_task_presets from isaaclab_tasks.utils.preset_target import PresetTarget from scripts.tools.train_and_publish_checkpoints import ( @@ -24,6 +26,34 @@ ) +def test_cartpole_feature_presets_are_in_pretrained_checkpoint_matrix() -> None: + """Every Cartpole feature policy for the preferred workflow must receive a distinct checkpoint.""" + task_spec = gym.spec("Isaac-Cartpole-Camera") + workflow = task_spec.kwargs["default_agent"] + feature_presets = task_spec.kwargs["agent_preset_compatibility"][f"{workflow}_feature_cfg_entry_point"] + + assert task_spec.kwargs["pretrained_checkpoint_preset_compatibility"][workflow] == feature_presets + + +def test_checkpoint_preset_metadata_references_registered_variants() -> None: + """Checkpoint declarations must name registered workflows and domain presets.""" + for task_spec in gym.registry.values(): + checkpoint_compatibility = task_spec.kwargs.get("pretrained_checkpoint_preset_compatibility", {}) + if not checkpoint_compatibility: + continue + + preset_map = enumerate_task_presets(task_spec.id) or {} + domain_presets = set(preset_map.get(PresetTarget.DOMAIN, ())) + for workflow, preset_names in checkpoint_compatibility.items(): + assert f"{workflow}_cfg_entry_point" in task_spec.kwargs, ( + f"{task_spec.id}: unregistered {workflow} workflow" + ) + assert len(preset_names) == len(set(preset_names)), ( + f"{task_spec.id}: duplicate {workflow} checkpoint preset" + ) + assert not set(preset_names) - domain_presets, f"{task_spec.id}: unknown {workflow} checkpoint preset" + + def test_build_core_jobs_skips_unsupported_preset_without_normalizing_default( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/source/isaaclab/changelog.d/flatten-image-features.rst b/source/isaaclab/changelog.d/flatten-image-features.rst new file mode 100644 index 000000000000..5aafd3529045 --- /dev/null +++ b/source/isaaclab/changelog.d/flatten-image-features.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Flattened multi-dimensional frozen-encoder outputs so they satisfy the observation-term contract and can be used + by MLP policies such as the Cartpole Theia-Tiny feature policy. diff --git a/source/isaaclab/isaaclab/envs/mdp/observations.py b/source/isaaclab/isaaclab/envs/mdp/observations.py index 4bdc05664686..65eb01f0f16c 100644 --- a/source/isaaclab/isaaclab/envs/mdp/observations.py +++ b/source/isaaclab/isaaclab/envs/mdp/observations.py @@ -557,8 +557,8 @@ def __call__( # forward the images through the model features = self._inference_fn(self._model, image_data, **(inference_kwargs or {})) - # move the features back to the image device - return features.detach().to(image_device) + # observation terms must be flat after the environment batch dimension + return features.flatten(start_dim=1).detach().to(image_device) """ Helper functions. diff --git a/source/isaaclab/test/envs/test_stacked_image_mdp.py b/source/isaaclab/test/envs/test_stacked_image_mdp.py index fcaa84a2034a..4e5f0c20d80a 100644 --- a/source/isaaclab/test/envs/test_stacked_image_mdp.py +++ b/source/isaaclab/test/envs/test_stacked_image_mdp.py @@ -19,7 +19,7 @@ pytestmark = pytest.mark.integration -from isaaclab.envs.mdp.observations import stacked_image +from isaaclab.envs.mdp.observations import image_features, stacked_image NUM_ENVS = 4 HEIGHT = 8 @@ -241,3 +241,17 @@ def test_clone_true_returns_independent_copy(self): cfg = SimpleNamespace(name="tiled_camera") out = image(env, sensor_cfg=cfg, data_type="rgb", normalize=False, clone=True) assert out.data_ptr() != camera_buf.data_ptr() + + +def test_image_features_flattens_encoder_output(): + """Feature extractors return a flat observation after the environment batch dimension.""" + env = _make_env() + term = image_features.__new__(image_features) + term._model = object() + term._inference_fn = lambda *_args, **_kwargs: torch.arange(NUM_ENVS * 6 * 8).reshape(NUM_ENVS, 6, 8) + image_data = torch.zeros((NUM_ENVS, HEIGHT, WIDTH, CHANNELS), dtype=torch.uint8) + + with mock.patch("isaaclab.envs.mdp.observations.image", return_value=image_data): + out = term(env) + + assert out.shape == (NUM_ENVS, 48) diff --git a/source/isaaclab_tasks/changelog.d/cartpole-feature-checkpoints.rst b/source/isaaclab_tasks/changelog.d/cartpole-feature-checkpoints.rst new file mode 100644 index 000000000000..0c44ed61ca24 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/cartpole-feature-checkpoints.rst @@ -0,0 +1,4 @@ +Fixed +^^^^^ + +* Added preset-specific RSL-RL checkpoint discovery for the ResNet18 and Theia-Tiny Cartpole camera policies. diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/__init__.py b/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/__init__.py index e914f87afbc7..4d5e0830fa22 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/__init__.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/cartpole/__init__.py @@ -24,6 +24,7 @@ "simple_shading_diffuse_mdl", "simple_shading_full_mdl", ) +_FEATURE_CAMERA_PRESETS = ("resnet18", "theia_tiny") ## # Register Gym environments -- direct workflow. @@ -93,9 +94,10 @@ ), "agent_preset_compatibility": { "rl_games_cfg_entry_point": _RAW_CAMERA_PRESETS, - "rl_games_feature_cfg_entry_point": ("resnet18", "theia_tiny"), + "rl_games_feature_cfg_entry_point": _FEATURE_CAMERA_PRESETS, "rsl_rl_cfg_entry_point": _RAW_CAMERA_PRESETS, - "rsl_rl_feature_cfg_entry_point": ("resnet18", "theia_tiny"), + "rsl_rl_feature_cfg_entry_point": _FEATURE_CAMERA_PRESETS, }, + "pretrained_checkpoint_preset_compatibility": {"rsl_rl": _FEATURE_CAMERA_PRESETS}, }, ) From 1326e940b2e6ca75591f11295f7dc24e7e48090a Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Tue, 8 Sep 2026 20:17:13 -0400 Subject: [PATCH 025/128] =?UTF-8?q?Cache=20marker=20scene-partition=20toke?= =?UTF-8?q?ns=20instead=20of=20rebuilding=20them=20each=20f=E2=80=A6=20(#7?= =?UTF-8?q?647)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description KitVisualizationMarkers.visualize() re-resolved and re-authored the primvars:omni:scenePartition token array on every call. Marker ownership is static in most tasks, but visualize() runs every frame — and ObjectUniformPoseCommand._update_metrics() calls it independently of debug_vis. At 4096 envs, every frame paid a device synchronization, a 4096-entry tuple rebuild, and 4096 token strings for a value that never changed. Isaac-Lift-KukaAllegro-Camera has two such instancers (SuccessMarkers, ObservationPointCloud). This caches the resolved tokens, the tensor they came from, and the tokens currently authored, so the primvar is rebuilt only when the environment IDs change. The partition-active probe caches its positive result; a negative one is still re-checked, since markers can be created before the renderer prepares the stage. Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. Fixes # (issue) ## Type of change - Bug fix (non-breaking change which fixes an issue) ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Benchmark Results Measured on `Isaac-Lift-KukaAllegro-Camera` (4096 envs, `duo_camera` preset, single L40 GPU), 200-step `isaaclab benchmark runtime` run before vs. after this commit on top of `develop` (25fad3eac7f): | Metric | Before | After | Change | | --- | --- | --- | --- | | Mean Total FPS | 3773.4 | 4270.6 | **+13.2%** | | Mean Iteration Time | 1085.5 ms | 959.1 ms | -11.6% | | Max Total FPS | 4284.9 | 4703.1 | +9.8% | | GPU Utilization | 28.6% | 37.3% | +8.7 pts | | Scene Creation Time | 295.9 s | 282.0 s | -4.7% | Single run per side, but well outside the ~150-180 FPS std-dev band reported within each run. Confirms the throughput improvement claimed above for this task's SuccessMarkers/ObservationPointCloud instancers. ## Screenshots Please attach before and after screenshots of the change if applicable. ## Checklist Docker and GPU tests run on demand. Push the commits you want tested, then comment `run-ci` on the pull request. - [ ] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [ ] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] I have added a changelog fragment under `source//changelog.d/` for every touched package (do **not** edit `CHANGELOG.rst` or bump `extension.toml` — CI handles that) - [ ] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- .../cache-marker-scene-partition.skip | 2 + .../markers/test_visualization_markers.py | 62 +++++++++++++++++++ .../cache-marker-scene-partition.rst | 9 +++ .../kit/kit_visualization_markers.py | 44 +++++++++++-- 4 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 source/isaaclab/changelog.d/cache-marker-scene-partition.skip create mode 100644 source/isaaclab_visualizers/changelog.d/cache-marker-scene-partition.rst diff --git a/source/isaaclab/changelog.d/cache-marker-scene-partition.skip b/source/isaaclab/changelog.d/cache-marker-scene-partition.skip new file mode 100644 index 000000000000..45ab2bc9a4b5 --- /dev/null +++ b/source/isaaclab/changelog.d/cache-marker-scene-partition.skip @@ -0,0 +1,2 @@ +Internal: added regression coverage for marker scene-partition token caching in +``isaaclab_visualizers``; no library behavior changed in this package. diff --git a/source/isaaclab/test/markers/test_visualization_markers.py b/source/isaaclab/test/markers/test_visualization_markers.py index ca7b1c4eaa4d..9af7d1fcafd3 100644 --- a/source/isaaclab/test/markers/test_visualization_markers.py +++ b/source/isaaclab/test/markers/test_visualization_markers.py @@ -204,6 +204,68 @@ def test_environment_ids_author_point_instance_scene_partitions(sim): assert list(primvar.Get()) == ["env_1", "env_0"] +def test_unchanged_environment_ids_do_not_rebuild_scene_partitions(sim, monkeypatch): + """Unchanged environment IDs should not rebuild partition tokens on every call. + + Marker ownership is static in most tasks, but ``visualize`` runs every frame. Rebuilding the + token array anyway costs a device synchronization and one string per marker per frame. + """ + from pxr import Sdf, UsdGeom, Vt + + sim._has_offscreen_render = True + stage = sim_utils.get_current_stage() + for env_id in range(2): + env_prim = stage.DefinePrim(f"/World/envs/env_{env_id}", "Xform") + env_prim.CreateAttribute("primvars:omni:scenePartition", Sdf.ValueTypeNames.Token).Set(f"env_{env_id}") + + config = VisualizationMarkersCfg( + prim_path="/World/Visuals/cached_partition_marker", + markers={"test": sim_utils.SphereCfg(radius=0.1)}, + ) + test_marker = VisualizationMarkers(config) + translations = torch.tensor([[0.0, 0.0, 0.0], [0.2, 0.0, 0.0]], device=sim.device) + environment_ids = torch.tensor([1, 0], device=sim.device) + test_marker.visualize(translations=translations, environment_ids=environment_ids) + + rebuilt_token_arrays = [] + original_token_array = Vt.TokenArray + + def _counting_token_array(*args, **kwargs): + rebuilt_token_arrays.append(args) + return original_token_array(*args, **kwargs) + + monkeypatch.setattr(Vt, "TokenArray", _counting_token_array) + # Markers move every frame while their environment ownership stays fixed. + test_marker.visualize(translations=translations + 0.1, environment_ids=environment_ids) + + assert rebuilt_token_arrays == [] + primvar = UsdGeom.PrimvarsAPI(stage.GetPrimAtPath(test_marker.prim_path)).GetPrimvar("omni:scenePartition") + assert list(primvar.Get()) == ["env_1", "env_0"] + + +def test_changed_environment_ids_reauthor_scene_partitions(sim): + """New environment IDs should still update the authored partition tokens.""" + from pxr import Sdf, UsdGeom + + sim._has_offscreen_render = True + stage = sim_utils.get_current_stage() + for env_id in range(2): + env_prim = stage.DefinePrim(f"/World/envs/env_{env_id}", "Xform") + env_prim.CreateAttribute("primvars:omni:scenePartition", Sdf.ValueTypeNames.Token).Set(f"env_{env_id}") + + config = VisualizationMarkersCfg( + prim_path="/World/Visuals/updated_partition_marker", + markers={"test": sim_utils.SphereCfg(radius=0.1)}, + ) + test_marker = VisualizationMarkers(config) + translations = torch.tensor([[0.0, 0.0, 0.0], [0.2, 0.0, 0.0]], device=sim.device) + test_marker.visualize(translations=translations, environment_ids=torch.tensor([1, 0], device=sim.device)) + test_marker.visualize(translations=translations, environment_ids=torch.tensor([0, 1], device=sim.device)) + + primvar = UsdGeom.PrimvarsAPI(stage.GetPrimAtPath(test_marker.prim_path)).GetPrimvar("omni:scenePartition") + assert list(primvar.Get()) == ["env_0", "env_1"] + + def test_environment_ids_require_active_scene_partitions(sim): """Environment IDs should not partition markers when renderer stage preparation is inactive.""" from pxr import UsdGeom diff --git a/source/isaaclab_visualizers/changelog.d/cache-marker-scene-partition.rst b/source/isaaclab_visualizers/changelog.d/cache-marker-scene-partition.rst new file mode 100644 index 000000000000..fd39e5c6e7ac --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/cache-marker-scene-partition.rst @@ -0,0 +1,9 @@ +Fixed +^^^^^ + +* Fixed :class:`~isaaclab_visualizers.kit.kit_visualization_markers.KitVisualizationMarkers` + rebuilding its scene-partition tokens on every frame. Marker ownership is now cached and the + ``primvars:omni:scenePartition`` primvar is only re-authored when the environment IDs change, + avoiding a device-to-host copy and one token string per marker on unchanged frames. A device + synchronization from comparing the cached and incoming environment IDs still occurs every call. + This noticeably improves throughput for camera tasks at high environment counts. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualization_markers.py b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualization_markers.py index 7aca82d5f029..0a08a12db7b4 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualization_markers.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualization_markers.py @@ -41,6 +41,14 @@ class KitVisualizationMarkers: .. _UsdGeom.PointInstancer: https://graphics.pixar.com/usd/dev/api/class_usd_geom_point_instancer.html """ + # Marker ownership is static in most tasks, but ``visualize`` runs every frame. These memoize + # the resolved partition tokens, the tensor they came from, and the tokens currently authored, + # so unchanged environment IDs skip the device-to-host copy, token rebuild, and USD write. The + # cached-vs-incoming tensor comparison itself still synchronizes the device every call. + _environment_ids_source: torch.Tensor | None = None + _authored_environment_ids: tuple[int, ...] | None = None + _scene_partitioning_active: bool = False + def __init__(self, cfg: VisualizationMarkersCfg, visible: bool = True): """Initialize the USD point instancer and register marker prototypes. @@ -146,11 +154,14 @@ def visualize( # changes and explicit marker indices are not provided. self._instancer_manager.GetProtoIndicesAttr().Set([0] * num_markers) if environment_ids is not None: - self._environment_ids = tuple(int(env_id) for env_id in environment_ids.detach().cpu().tolist()) + if not self._matches_cached_environment_ids(environment_ids): + self._environment_ids_source = environment_ids.detach().clone() + self._environment_ids = tuple(int(env_id) for env_id in self._environment_ids_source.cpu().tolist()) if num_markers == 0: num_markers = len(self._environment_ids) elif num_markers != 0 and num_markers != previous_count: self._environment_ids = None + self._environment_ids_source = None if num_markers != 0: self._count = num_markers self._sync_scene_partition_primvar() @@ -164,12 +175,20 @@ def _sync_scene_partition_primvar(self) -> None: """ from pxr import Sdf, UsdGeom, Vt # noqa: PLC0415 + target_ids = self._environment_ids if self._scene_partitioning_is_active() else None + # The tuple is rebuilt only when marker ownership changes, so identity is enough to tell + # that the authored tokens are still current. Skipping the write keeps the renderer from + # rebuilding its partition data on frames where nothing moved between environments. + if target_ids is self._authored_environment_ids: + return + primvars_api = UsdGeom.PrimvarsAPI(self._instancer_manager.GetPrim()) # PrimvarsAPI adds the ``primvars:`` namespace, matching the env-root attribute. primvar = primvars_api.GetPrimvar("omni:scenePartition") - if self._environment_ids is None or not self._scene_partitioning_is_active(): + if target_ids is None: if primvar: primvar.GetAttr().Clear() + self._authored_environment_ids = None return if not primvar: @@ -184,7 +203,19 @@ def _sync_scene_partition_primvar(self) -> None: raise RuntimeError( f"Expected '{primvar.GetAttr().GetPath()}' to have type TokenArray. Received: {primvar.GetTypeName()}." ) - primvar.Set(Vt.TokenArray([f"env_{env_id}" for env_id in self._environment_ids])) + primvar.Set(Vt.TokenArray([f"env_{env_id}" for env_id in target_ids])) + self._authored_environment_ids = target_ids + + def _matches_cached_environment_ids(self, environment_ids: torch.Tensor) -> bool: + """Return whether ``environment_ids`` still describes the cached marker ownership.""" + cached = self._environment_ids_source + return ( + cached is not None + and cached.shape == environment_ids.shape + and cached.dtype == environment_ids.dtype + and cached.device == environment_ids.device + and bool(torch.equal(cached, environment_ids)) + ) def _scene_partitioning_is_active(self) -> bool: """Return whether renderer stage preparation authored environment partitions. @@ -192,11 +223,16 @@ def _scene_partitioning_is_active(self) -> bool: Renderer preparation always starts with ``env_0``, so its root is the canonical stage-level signal regardless of which environments own markers. """ + # Markers can be created before the renderer prepares the stage, so a negative result is + # re-checked. Partitions are never withdrawn from a stage, so a positive one is cached. + if self._scene_partitioning_active: + return True env_prim = self.stage.GetPrimAtPath("/World/envs/env_0") if not env_prim.IsValid(): return False attr = env_prim.GetAttribute("primvars:omni:scenePartition") - return attr.IsValid() and attr.Get() is not None + self._scene_partitioning_active = attr.IsValid() and attr.Get() is not None + return self._scene_partitioning_active def _add_markers_prototypes(self, markers_cfg: dict[str, sim_utils.SpawnerCfg]) -> None: """Add marker prototypes to the scene and register them with the point instancer.""" From 4db018695a97cc127a1b664827bdbca1a5d0ae7c Mon Sep 17 00:00:00 2001 From: Mustafa H <34825877+StafaH@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:18:56 -0400 Subject: [PATCH 026/128] [Docs] Use uv pip for Python package installation (#7643) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Updates the CUDA-enabled PyTorch commands in the Isaac Lab Python package installation workflow to use `uv pip`, matching the uv-based environment setup. The managed conda workflow continues to render `python -m pip`. No new dependencies are required. ## Type of change - Bug fix (non-breaking change which fixes an issue) - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not applicable. ## Validation - `uv run --isolated --extra dev -- sphinx-build -W --keep-going -j auto docs ` — passed without warnings. - Verified the rendered Python-package commands use `uv pip` for Linux x86_64, Windows x86_64, and Linux aarch64. - `uv run isaaclab -f` — all applicable formatting and documentation hooks passed. The repository-wide changelog hook reports unrelated existing `develop` discrepancies in packages untouched by this docs-only change. ## Checklist - [x] I have read and understood the contribution guidelines - [x] I have run the applicable pre-commit checks - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] No source package was changed, so no changelog fragment is required - [x] My name already exists in `CONTRIBUTORS.md` --- docs/source/setup/installation/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/setup/installation/index.rst b/docs/source/setup/installation/index.rst index a153cd81613f..b81cdc66fa3e 100644 --- a/docs/source/setup/installation/index.rst +++ b/docs/source/setup/installation/index.rst @@ -721,12 +721,12 @@ Install the CUDA-enabled PyTorch build appropriate for your system architecture: .. tab-item:: :icon:`fa-brands fa-linux` Linux (x86_64) :sync: linux-x86_64 - .. isaaclab-torch-install:: cu128 pip + .. isaaclab-torch-install:: cu128 .. tab-item:: :icon:`fa-brands fa-windows` Windows (x86_64) :sync: windows-x86_64 - .. isaaclab-torch-install:: cu128 pip + .. isaaclab-torch-install:: cu128 .. tab-item:: :icon:`fa-brands fa-linux` Linux (aarch64) :sync: linux-aarch64 @@ -741,7 +741,7 @@ Install the CUDA-enabled PyTorch build appropriate for your system architecture: sudo apt install python3.12-dev libgl1-mesa-dev libx11-dev libxcursor-dev libxi-dev \ libxinerama-dev libxrandr-dev - .. isaaclab-torch-install:: cu130 pip + .. isaaclab-torch-install:: cu130 .. note:: From e591032ce4fc17cbce14679afdb37b960b149ac5 Mon Sep 17 00:00:00 2001 From: Mustafa H <34825877+StafaH@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:24:02 -0400 Subject: [PATCH 027/128] [Docs] Improve environment browser cards (#7644) # Description Improve the environment browser by presenting related task variants as visual cards and making task capabilities easier to discover. This change: - groups manager-based, direct, camera, and direct-camera variants into one card - adds preview images and physics, renderer, and RL capability badges - supports searching by task name or capability - moves the Core, Contrib, and Warp scope selector alongside the task filters - updates responsive styling for the new card layout No additional dependencies are required. ## Type of change - Documentation update ## Release backport - [x] Backport this pull request to the active release branch after it merges into `develop` ## Screenshots Not included; the rendered documentation preview shows the updated card layout. ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the formatting and pre-commit checks - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [x] I have added a changelog fragment under `source//changelog.d/` for every touched package (no source packages are touched) - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --- docs/conf.py | 8 +- .../_static/css/environment-browser.css | 144 +++++++++--- .../source/_static/css/environment-browser.js | 208 +++++++++++++++--- docs/source/setup/environments.rst | 14 +- 4 files changed, 290 insertions(+), 84 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1c53a431424d..36c1b64a9487 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -325,13 +325,7 @@ def _read_pinned_versions() -> dict: # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [ - "source/_static/css", - "source/_static/how-to", - "source/_static/visualizers", - "source/_static/tasks/previews", - "source/_static/benchmarks", -] +html_static_path = ["source/_static"] html_css_files = ["custom.css", "environment-browser.css"] html_js_files = ["environment-browser.js"] diff --git a/docs/source/_static/css/environment-browser.css b/docs/source/_static/css/environment-browser.css index 47e73ba9f107..75941fcb85f4 100644 --- a/docs/source/_static/css/environment-browser.css +++ b/docs/source/_static/css/environment-browser.css @@ -410,61 +410,138 @@ html[data-theme="dark"] .environment-browser { .environment-task-list { display: grid; - max-height: 31rem; - overflow-y: auto; + grid-template-columns: repeat(auto-fill, minmax(12.5rem, 1fr)); + gap: 0.75rem; +} + +.environment-task-card { + display: flex; + flex-direction: column; + min-width: 0; + overflow: hidden; border: 1px solid var(--environment-border); border-radius: 8px; + color: var(--pst-color-text-base); + background: var(--pst-color-background); } -.environment-task-row { +.environment-task-card:hover, +.environment-task-card:has(.environment-task-card-select:focus-visible) { + border-color: var(--pst-color-primary); + box-shadow: 0 6px 16px color-mix(in srgb, var(--pst-color-shadow) 18%, transparent); +} + +.environment-task-card.is-selected { + border-color: var(--pst-color-primary); + box-shadow: 0 0 0 2px color-mix(in srgb, var(--pst-color-primary) 35%, transparent); +} + +.environment-task-card-select { display: grid; - grid-template-columns: minmax(0, 1fr) auto; - align-items: center; - gap: 1rem; + grid-template-rows: auto 1fr; + align-content: start; width: 100%; - min-height: 3.4rem; - padding: 0.65rem 0.85rem; + min-width: 0; + padding: 0; border: 0; - border-bottom: 1px solid var(--environment-border); - border-radius: 0; - color: var(--pst-color-text-base); - background: var(--pst-color-background); + color: inherit; + background: transparent; + cursor: pointer; text-align: left; } -.environment-task-row:last-child { - border-bottom: 0; -} - -.environment-task-row:hover, -.environment-task-row:focus-visible, -.environment-task-row.is-selected { - background: color-mix(in srgb, var(--pst-color-primary) 9%, var(--pst-color-background)); +.environment-task-card-select > img { + display: block; + width: 100%; + aspect-ratio: 16 / 10; + border-bottom: 1px solid var(--environment-border); + object-fit: cover; } -.environment-task-row.is-selected { - box-shadow: inset 3px 0 var(--pst-color-primary); +.environment-task-card-content { + display: grid; + align-content: start; + min-width: 0; + padding: 0.55rem 0.6rem 0.35rem; } .environment-task-name { + display: block; min-width: 0; overflow-wrap: anywhere; font-family: var(--pst-font-family-monospace); + font-size: 0.8rem; font-weight: 650; } -.environment-task-meta { +.environment-task-symbols { + display: flex; + flex-wrap: wrap; + gap: 0.3rem; + padding: 0 0.6rem 0.6rem; +} + +.environment-task-symbols:empty { + display: none; +} + +.environment-task-symbol { display: inline-flex; - gap: 0.45rem; - color: var(--environment-muted); - font-size: 0.78rem; - white-space: nowrap; + align-items: center; + gap: 0.25rem; + padding: 0.18rem 0.45rem; + border: 1px solid transparent; + border-radius: 999px; + font-family: var(--pst-font-family-monospace); + font-size: 0.64rem; + font-weight: 600; + line-height: 1.25; +} + +.environment-task-symbol i { + font-size: 0.6rem; } -.environment-task-meta span { - padding: 0.15rem 0.4rem; +.environment-task-symbol-physics { + border-color: color-mix(in srgb, var(--environment-physics) 35%, transparent); + color: var(--environment-physics); + background: color-mix(in srgb, var(--environment-physics) 8%, transparent); +} + +.environment-task-symbol-renderer { + border-color: color-mix(in srgb, var(--environment-renderer) 35%, transparent); + color: var(--environment-renderer); + background: color-mix(in srgb, var(--environment-renderer) 8%, transparent); +} + +.environment-task-symbol-rl { + border-color: color-mix(in srgb, var(--environment-preset) 35%, transparent); + color: var(--environment-preset); + background: color-mix(in srgb, var(--environment-preset) 8%, transparent); +} + +.environment-task-variants { + display: flex; + flex-wrap: wrap; + gap: 0.25rem; + padding: 0 0.6rem 0.45rem; +} + +.environment-task-variants button { + padding: 0.12rem 0.45rem; border: 1px solid var(--environment-border); - border-radius: 4px; + border-radius: 999px; + color: var(--environment-muted); + background: color-mix(in srgb, var(--pst-color-surface) 65%, transparent); + font-size: 0.68rem; + font-weight: 600; + cursor: pointer; +} + +.environment-task-variants button.is-selected { + color: #1f3300; + border-color: #76b900; + background: #76b900; } .environment-empty-state { @@ -745,13 +822,8 @@ html[data-theme="dark"] .environment-browser { width: 100%; } - .environment-task-row { + .environment-task-list { grid-template-columns: 1fr; - gap: 0.35rem; - } - - .environment-task-meta { - white-space: normal; } .environment-benchmark-panel { diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index 54835689ec66..581a45c7e982 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -187,12 +187,13 @@ const copyButton = builder.querySelector("[data-copy-command]"); const copyStatus = builder.querySelector("[data-copy-status]"); const modeButtons = [...builder.querySelectorAll("[data-command-mode]")]; - const scopeButtons = [...builder.querySelectorAll("[data-task-scope]")]; + const scopeButtons = [...taskBrowser.querySelectorAll("[data-task-scope]")]; const taskList = taskBrowser.querySelector("[data-task-list]"); const taskSearch = taskBrowser.querySelector("[data-task-search]"); const taskCategory = taskBrowser.querySelector("[data-task-category]"); const taskCount = taskBrowser.querySelector("[data-task-count]"); const taskEmpty = taskBrowser.querySelector("[data-task-empty]"); + const taskCardRefreshers = new WeakMap(); const state = { mode: "train", scope: "core", @@ -212,6 +213,89 @@ return "classic"; }; + // Direct/camera variants of the same task are folded into a single card; this only strips + // exact trailing suffixes, so unrelated tasks that merely contain "Camera"/"Direct" elsewhere + // in their name are never merged. + const variantOrder = ["manager", "direct", "camera", "direct-camera"]; + const variantLabels = {manager: "Manager", direct: "Direct", camera: "Camera", "direct-camera": "Direct-Camera"}; + const variantOf = (task) => { + if (task.endsWith("-Camera-Direct")) { + return "direct-camera"; + } + if (task.endsWith("-Direct")) { + return "direct"; + } + if (task.endsWith("-Camera")) { + return "camera"; + } + return "manager"; + }; + const baseTaskName = (task) => task + .replace(/-Camera-Direct$/, "") + .replace(/-Direct$/, "") + .replace(/-Camera$/, ""); + + const groupTasks = (taskList) => { + const groups = new Map(); + for (const task of taskList) { + const base = baseTaskName(task.task); + if (!groups.has(base)) { + groups.set(base, []); + } + groups.get(base).push(task); + } + for (const variants of groups.values()) { + variants.sort((left, right) => ( + variantOrder.indexOf(variantOf(left.task)) - variantOrder.indexOf(variantOf(right.task)) + )); + } + return groups; + }; + + // Keep capability labels compact on the cards while retaining the full names in tooltips and + // accessible labels. + const capabilitySymbolSets = [ + ["physics", [ + ["isaacsim_physx", "physx", "Isaac Sim PhysX"], + ["newton_kamino", "kamino", "Newton Kamino"], + ["newton_mjwarp", "mjwarp", "Newton MJWarp"], + ["newton_mjwarp_vbd_proxy", "mjwarp vbd", "Newton MJWarp VBD proxy"], + ["ovphysx", "ovphysx", "OV PhysX"], + ]], + ["renderer", [ + ["isaacsim_rtx", "rtx", "Isaac Sim RTX"], + ["newton_renderer", "renderer", "Newton renderer"], + ["ovrtx", "ovrtx", "OV RTX"], + ]], + ["rl", [ + ["rl_games", "rl_games", "RL Games"], + ["rsl_rl", "rsl_rl", "RSL-RL"], + ["skrl", "skrl", "skrl"], + ["sb3", "sb3", "Stable-Baselines3"], + ["rlinf", "rlinf", "RLinf"], + ]], + ]; + + const buildCapabilitySymbols = (task) => { + const values = {physics: task.physics, renderer: task.renderer, rl: task.rl}; + const icons = {physics: "fa-gears", renderer: "fa-eye", rl: "fa-chart-line"}; + return capabilitySymbolSets.flatMap(([type, symbols]) => symbols + .filter(([value]) => values[type].includes(value)) + .map(([, shortLabel, fullLabel]) => { + const symbol = document.createElement("span"); + symbol.className = `environment-task-symbol environment-task-symbol-${type}`; + symbol.title = fullLabel; + symbol.setAttribute("aria-label", fullLabel); + const icon = document.createElement("i"); + icon.className = `fa-solid ${icons[type]}`; + icon.setAttribute("aria-hidden", "true"); + const label = document.createElement("span"); + label.textContent = shortLabel; + symbol.replaceChildren(icon, label); + return symbol; + })); + }; + const preferredValue = (values, preferred) => preferred.find((value) => values.includes(value)) || values[0] || ""; const populateSelect = (select, values, preferred) => { @@ -251,7 +335,6 @@ [/Reorient-Franka/, "tasks/manipulation/franka_lift.jpg"], [/Reorient-KukaAllegro/, "tasks/manipulation/kuka_allegro_reorient.jpg"], [/Shadow-Handover/, "tasks/manipulation/shadow_hand_over.jpg"], - [/Keyboard-SO101/, "tasks/manipulation/so101_keyboard.jpg"], [/AnymalB/, "tasks/locomotion/anymal_b_flat.jpg"], [/AnymalC/, "tasks/locomotion/anymal_c_flat.jpg"], [/AnymalD/, "tasks/locomotion/anymal_d_flat.jpg"], @@ -415,6 +498,9 @@ updateModeControls(); commandOutput.textContent = currentCommand(); updatePreview(); + for (const card of taskList.querySelectorAll(".environment-task-card")) { + taskCardRefreshers.get(card)?.(); + } for (const row of taskList.querySelectorAll("[data-task-name]")) { const isSelected = row.dataset.taskName === state.task; row.classList.toggle("is-selected", isSelected); @@ -422,44 +508,98 @@ } }; + const createTaskCard = (variants) => { + const card = document.createElement("div"); + card.className = "environment-task-card"; + + const selectButton = document.createElement("button"); + selectButton.type = "button"; + selectButton.className = "environment-task-card-select"; + + const image = document.createElement("img"); + image.alt = ""; + image.loading = "lazy"; + + const content = document.createElement("span"); + content.className = "environment-task-card-content"; + const nameEl = document.createElement("span"); + nameEl.className = "environment-task-name"; + const symbolsEl = document.createElement("span"); + symbolsEl.className = "environment-task-symbols"; + content.append(nameEl); + selectButton.append(image, content); + + const variantsRow = document.createElement("div"); + variantsRow.className = "environment-task-variants"; + variantsRow.setAttribute("role", "group"); + variantsRow.setAttribute("aria-label", "Task variant"); + for (const variantTask of variants) { + const variantButton = document.createElement("button"); + variantButton.type = "button"; + variantButton.textContent = variantLabels[variantOf(variantTask.task)]; + variantButton.dataset.taskName = variantTask.task; + variantButton.addEventListener("click", (event) => { + event.stopPropagation(); + state.task = variantTask.task; + refreshCard(); + updateSelection(); + }); + variantsRow.append(variantButton); + } + + const refreshCard = () => { + const activeTask = variants.find((task) => task.task === state.task) || variants[0]; + const isSelected = variants.includes(activeTask) && activeTask.task === state.task; + image.src = new URL(`../../_static/${previewImageFor(activeTask)}`, window.location.href).href; + image.alt = ""; + selectButton.dataset.taskName = activeTask.task; + selectButton.setAttribute("aria-pressed", String(isSelected)); + nameEl.textContent = activeTask.task; + symbolsEl.replaceChildren(...buildCapabilitySymbols(activeTask)); + card.classList.toggle("is-selected", isSelected); + for (const button of variantsRow.querySelectorAll("[data-task-name]")) { + const isActiveVariant = button.dataset.taskName === state.task; + button.classList.toggle("is-selected", isActiveVariant); + button.setAttribute("aria-pressed", String(isActiveVariant)); + } + }; + + selectButton.addEventListener("click", () => { + state.task = selectButton.dataset.taskName; + updateSelection(); + }); + + taskCardRefreshers.set(card, refreshCard); + refreshCard(); + card.append(selectButton, variantsRow, symbolsEl); + return card; + }; + const renderTasks = () => { const query = taskSearch.value.trim().toLowerCase(); const category = taskCategory.value; - const visibleTasks = tasksForScope().filter((task) => { - const matchesQuery = task.task.toLowerCase().includes(query); + const matchesFilter = (task) => { + const searchableValues = [ + task.task, + ...(task.physics.length ? task.physics : ["Default"]), + ...(task.renderer.length ? task.renderer : ["Default"]), + ...(task.rl.length ? task.rl : ["Not supported"]), + ]; + const matchesQuery = searchableValues.some((value) => value.toLowerCase().includes(query)); const matchesCategory = category === "all" || categoryFor(task.task) === category; return matchesQuery && matchesCategory; - }); - taskList.replaceChildren(...visibleTasks.map((task) => { - const button = document.createElement("button"); - button.type = "button"; - button.className = "environment-task-row"; - button.dataset.taskName = task.task; - const isSelected = task.task === state.task; - button.classList.toggle("is-selected", isSelected); - button.setAttribute("aria-pressed", String(isSelected)); - button.innerHTML = ``; - button.querySelector(".environment-task-name").textContent = task.task; - const meta = button.querySelector(".environment-task-meta"); - const workflow = task.task.includes("Direct") ? "Direct" : "Manager based"; - const rlSupport = task.rl.length - ? `${task.rl.length} RL ${task.rl.length === 1 ? "library" : "libraries"}` - : "RL not supported"; - meta.replaceChildren(...[workflow, rlSupport].map((label) => { - const badge = document.createElement("span"); - badge.textContent = label; - return badge; - })); - button.addEventListener("click", () => { - state.task = task.task; - updateSelection(); - }); - return button; - })); - taskCount.textContent = `${visibleTasks.length} ${visibleTasks.length === 1 ? "task" : "tasks"}`; - taskEmpty.hidden = visibleTasks.length !== 0; - taskList.hidden = visibleTasks.length === 0; + }; + const groups = groupTasks(tasksForScope()); + const visibleGroups = [...groups.values()] + .map((variants) => variants.filter(matchesFilter)) + .filter((variants) => variants.length > 0); + + taskList.replaceChildren(...visibleGroups.map(createTaskCard)); + const matchingTaskCount = visibleGroups.reduce((total, variants) => total + variants.length, 0); + taskCount.textContent = `${matchingTaskCount} ${matchingTaskCount === 1 ? "task" : "tasks"}`; + taskEmpty.hidden = visibleGroups.length !== 0; + taskList.hidden = visibleGroups.length === 0; }; const initializeTasks = () => { diff --git a/docs/source/setup/environments.rst b/docs/source/setup/environments.rst index 1ca42117394c..d6f4c3abde10 100644 --- a/docs/source/setup/environments.rst +++ b/docs/source/setup/environments.rst @@ -31,11 +31,6 @@ Command Builder --task -
- - - -