Add dVRK needle-pass environment and OpenXR teleoperation

.github/actions/run-tests/action.yml

@@ -31,6 +31,10 @@ inputs:
     description: 'Pattern to filter test files (e.g., isaaclab_tasks)'
     default: ''
     required: false
+  install-isaacteleop-pr769:
+    description: 'Build and install the test-only IsaacTeleop PR 769 merge revision before pytest'
+    default: 'false'
+    required: false
 
 runs:
   using: composite
@@ -47,6 +51,7 @@ runs:
           local reports_dir="$5"
           local pytest_options="$6"
           local filter_pattern="$7"
+          local install_isaacteleop_pr769="$8"
 
           echo "Running tests in: $test_path"
           if [ -n "$pytest_options" ]; then
@@ -106,12 +111,20 @@ runs:
               set -e
               cd /workspace/isaaclab
               mkdir -p tests
+              if [ \"$install_isaacteleop_pr769\" = \"true\" ]; then
+                bash scripts/tools/install_isaacteleop_pr769_for_tests.sh
+              fi
               echo 'Starting pytest with path: $test_path'
-              /isaac-sim/python.sh -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v --junitxml=tests/$result_file || echo 'Pytest completed with exit code: $?'
+              # tools/conftest.py dispatches each Isaac Sim test in its own process,
+              # applies TEST_FILTER_PATTERN, and combines the per-test reports.  It
+              # must be loaded: ignoring it would leave the CI jobs with no
+              # meaningful task/device test coverage.
+              /isaac-sim/python.sh -m pytest $test_path $pytest_options -v --junitxml=tests/$result_file
             "; then
             echo "✅ Docker container completed successfully"
           else
-            echo "⚠️ Docker container failed, but continuing to copy results..."
+            test_status=$?
+            echo "❌ Docker container failed; preserving reports before failing this action."
           fi
 
           # Copy test results with error handling
@@ -151,7 +164,8 @@ runs:
           # Clean up container
           echo "🧹 Cleaning up Docker container..."
           docker rm $container_name 2>/dev/null || echo "⚠️ Container cleanup failed, but continuing..."
+          return ${test_status:-0}
         }
 
         # Call the function with provided parameters
-        run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}"
+        run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.install-isaacteleop-pr769 }}"

.github/workflows/build.yml

@@ -33,7 +33,6 @@ jobs:
   test-isaaclab-tasks:
     runs-on: [self-hosted, gpu]
     timeout-minutes: 180
-    continue-on-error: true
 
     steps:
     - name: Checkout Code
@@ -118,6 +117,7 @@ jobs:
         image-tag: ${{ env.DOCKER_IMAGE_TAG }}
         pytest-options: ""
         filter-pattern: "not isaaclab_tasks"
+        install-isaacteleop-pr769: true
 
     - name: Copy Test Results from General Tests Container
       run: |

CONTRIBUTORS.md

@@ -60,6 +60,7 @@ Guidelines for modifications:
 * Cathy Y. Li
 * Cheng-Rong Lai
 * Chenyu Yang
+* Chris von Csefalvay
 * Connor Smith
 * CY (Chien-Ying) Chen
 * David Yang

docs/source/api/lab/isaaclab.devices.rst

@@ -17,7 +17,12 @@
     Se3SpaceMouse
     HaplyDevice
     OpenXRDevice
+    DVRKOpenXRDevice
+    DVRKOpenXRDeviceCfg
     ManusVive
+    isaaclab.devices.openxr.retargeters.DVRKPSMRetargeter
+    isaaclab.devices.openxr.retargeters.DVRKPSMRetargeterCfg
+    isaaclab.devices.openxr.retargeters.DVRKPSMSideRetargeterCfg
     isaaclab.devices.openxr.retargeters.GripperRetargeter
     isaaclab.devices.openxr.retargeters.Se3AbsRetargeter
     isaaclab.devices.openxr.retargeters.Se3RelRetargeter
@@ -104,6 +109,20 @@ OpenXR
     :show-inheritance:
     :noindex:
 
+dVRK OpenXR
+-----------
+
+.. autoclass:: DVRKOpenXRDevice
+    :members:
+    :inherited-members:
+    :show-inheritance:
+    :noindex:
+
+.. autoclass:: DVRKOpenXRDeviceCfg
+    :members:
+    :show-inheritance:
+    :noindex:
+
 Manus + Vive
 ------------
 
@@ -116,6 +135,22 @@ Manus + Vive
 Retargeters
 -----------
 
+.. autoclass:: isaaclab.devices.openxr.retargeters.DVRKPSMRetargeter
+    :members:
+    :inherited-members:
+    :show-inheritance:
+    :noindex:
+
+.. autoclass:: isaaclab.devices.openxr.retargeters.DVRKPSMRetargeterCfg
+    :members:
+    :show-inheritance:
+    :noindex:
+
+.. autoclass:: isaaclab.devices.openxr.retargeters.DVRKPSMSideRetargeterCfg
+    :members:
+    :show-inheritance:
+    :noindex:
+
 .. autoclass:: isaaclab.devices.openxr.retargeters.GripperRetargeter
     :members:
     :inherited-members:

docs/source/how-to/cloudxr_teleoperation.rst

@@ -721,6 +721,72 @@ Isaac Lab provides three main retargeters for hand tracking:
 
 Retargeters can be combined to control different robot functions simultaneously.
 
+dVRK bimanual PSM retargeting
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The :class:`isaaclab.devices.DVRKOpenXRDevice` and
+:class:`isaaclab.devices.openxr.retargeters.DVRKPSMRetargeter` provide a reusable
+motion-controller interface for two da Vinci Research Kit (dVRK) Patient Side
+Manipulators (PSMs). Isaac Lab remains the owner of the OpenXR session and raw
+controller source. The retargeter wraps the simulator-independent Cartesian-clutch
+and jaw-intent state machines supplied by the ``isaacteleop`` package; it does not
+construct a second Isaac Teleop retargeting graph.
+
+The ``isaacteleop`` installation must expose these public classes from
+``isaacteleop.retargeters.DVRK.control``:
+
+* ``DVRKPSMCartesianClutchConfig`` and ``DVRKPSMCartesianClutchStateMachine``
+* ``DVRKPSMJawIntentConfig`` and ``DVRKPSMJawIntentStateMachine``
+
+Isaac Lab imports them only when the dVRK retargeter is constructed, so task
+discovery still works when the optional runtime is unavailable. Construction
+raises a targeted installation error if the required classes are missing.
+
+The controller mapping is fixed:
+
+* the left controller commands the left PSM and the right controller commands the right PSM;
+* the grip pose commands the corresponding tool-tip pose;
+* squeeze is the clutch and deadman control;
+* the index trigger commands paired-jaw intent; and
+* thumbsticks and face buttons are not consumed.
+
+Each controller must provide a valid grip position and orientation. Missing,
+malformed, non-finite, or partially valid data holds that side's complete command
+while the other side continues independently; inspect ``controller_faults`` on
+the OpenXR device before continuing a bimanual procedure. Unexpected XR runtime
+errors propagate instead of being silently treated as tracking loss. Isaac Lab converts the OpenXR
+scalar-first ``wxyz`` quaternion to Isaac Teleop's scalar-last ``xyzw`` convention.
+The returned contiguous ``float32`` action has 18 values in this stable order:
+
+.. code-block:: text
+
+   [left position xyz, left quaternion xyzw, left jaw_1, left jaw_2,
+    right position xyz, right quaternion xyzw, right jaw_1, right jaw_2]
+
+Configure each :class:`isaaclab.devices.openxr.retargeters.DVRKPSMSideRetargeterCfg`
+with an explicit world-frame home pose, workspace bounds, orientation calibration,
+and measured open and closed jaw endpoints. The controller stream, home, and bounds
+share the world/XR command frame. A bimanual task must convert each world-frame
+tool target through that PSM's current root transform immediately before its IK
+solve; the two independently placed PSMs must not share one cached root transform.
+
+``initial_closedness`` is a per-side reset target. The dVRK needle-pass task deliberately
+uses asymmetric values: the donor restores its load-qualified closed grasp around the
+needle, while the receiver restores fully open. ``RESET`` returns the controller command
+to the same donor-held and receiver-open state as the task articulation.
+
+Session lifecycle is explicit. ``START`` activates both state machines and the first
+valid squeezed sample captures fresh controller origins without moving either tool.
+``STOP`` deactivates and disengages the state machines while preserving the last
+pose and jaw targets. ``RESET`` restores the configured home targets and fresh-origin
+requirement without changing whether the session is active. Set
+``teleoperation_active_default=False`` when an application requires the operator to
+choose **Start Teleop** before inputs can change the target.
+
+Command hold is limited to this adapter's output. It does not disable actuator
+drives, latch measured joint state, create a grasp constraint, or attach a manipulated
+object.
+
 Using Retargeters with Hand Tracking
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

docs/source/overview/imitation-learning/teleop_imitation.rst

@@ -114,6 +114,65 @@ To collect demonstrations with teleoperation for the environment ``Isaac-Stack-C
    # step a: replay the collected dataset
    ./isaaclab.sh -p scripts/tools/replay_demos.py --task Isaac-Stack-Cube-Franka-IK-Rel-v0 --device cpu --dataset_file ./datasets/dataset.hdf5
 
+The bimanual dVRK needle-pass task uses the same stock scripts with its paired Quest motion controllers.
+It references the dVRK PSM, SDF suture needle, and suture pad from the Isaac for Healthcare
+``0.6.0`` asset catalogue at content revision ``c189487``; the remote assets are not redistributed
+with Isaac Lab. Each reset begins with the free dynamic needle physically held between the closed donor
+jaws while the receiver jaws remain open. The operator therefore starts by preserving the donor hold,
+then acquires the receiver grasp and releases the donor; the task does not begin with an ungrasped needle.
+After configuring CloudXR as described in :ref:`cloudxr-teleoperation`, first check manual control with
+the stock teleoperation entry point. Asset digest verification is an explicit
+online preflight so task discovery and configuration remain offline-safe:
+
+.. code:: bash
+
+   ./isaaclab.sh -p scripts/tools/preflight_dvrk_needle_pass_assets.py
+
+   ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \
+      --task Isaac-NeedlePass-dVRK-IK-Abs-v0 \
+      --teleop_device motion_controllers \
+      --device cuda:0 --xr
+
+Then record and replay one successful demonstration with:
+
+.. code:: bash
+
+   # record one successful demonstration
+   ./isaaclab.sh -p scripts/tools/record_demos.py \
+      --task Isaac-NeedlePass-dVRK-IK-Abs-v0 \
+      --teleop_device motion_controllers \
+      --dataset_file ./datasets/dvrk_needle_pass.hdf5 \
+      --num_demos 1 --num_success_steps 10 --step_hz 30 --device cuda:0 --xr
+
+   # replay the recorded demonstration
+   ./isaaclab.sh -p scripts/tools/replay_demos.py \
+      --task Isaac-NeedlePass-dVRK-IK-Abs-v0 \
+      --dataset_file ./datasets/dvrk_needle_pass.hdf5 \
+      --num_envs 1 --device cuda:0 --validate_success_rate
+
+The needle remains a free dynamic rigid body throughout the episode. Reset is
+the only operation that writes its pose or velocity; donor hold, co-hold,
+receiver-only hold, and retained lift are advanced from filtered bilateral jaw
+contact and simulated needle motion. The recorder therefore exports success
+only after a contact-driven transfer and retained receiver lift, without a
+grasp joint, attachment, or scripted object following.
+
+The dVRK needle-pass task requires CUDA PhysX.  CPU is intentionally rejected:
+the contact-qualified grasp and hand-off contracts are validated only on the
+CUDA simulation path.
+
+To retain a reviewable CUDA-physics video of the one-environment runtime
+verification trace, set an output directory before running its test:
+
+.. code-block:: bash
+
+   ISAACLAB_DVRK_NEEDLE_PASS_VIDEO_DIR=$PWD/output/dvrk-needle-pass \
+     ./isaaclab.sh -p -m pytest \
+       source/isaaclab_tasks/test/test_dvrk_needle_pass_physics.py \
+       -k native_grasp_generator_handoff_video -q
+
+The test fails if Gymnasium's ``RecordVideo`` wrapper does not write the MP4.
+
 
 .. note::
 

scripts/tools/install_isaacteleop_pr769_for_tests.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Test-only pin: this merge revision includes NVIDIA/IsaacTeleop PR 769.  This
+# PR is part of the dVRK Teleop Extended Universe and, once merged, this pin
+# will be updated to align with the corresponding released IsaacTeleop version.
+set -euo pipefail
+
+readonly ISAAC_TELEOP_REPOSITORY="https://github.com/NVIDIA/IsaacTeleop.git"
+readonly ISAAC_TELEOP_PR769_MERGE_SHA="790d6cb4e948de377975c76ed1e9cbf5098e10fc"
+readonly ISAACLAB_PYTHON="${ISAACLAB_PATH:?ISAACLAB_PATH must be set}/_isaac_sim/python.sh"
+readonly ISAACLAB_PYTHON_SCRIPTS="$("${ISAACLAB_PYTHON}" -c 'import sysconfig; print(sysconfig.get_path("scripts"))')"
+readonly BUILD_ROOT="${ISAACLAB_TELEOP_TEST_CACHE:-/tmp/isaacteleop-pr769-${ISAAC_TELEOP_PR769_MERGE_SHA}}"
+readonly SOURCE_DIR="${BUILD_ROOT}/source"
+readonly CMAKE_BUILD_DIR="${BUILD_ROOT}/build"
+readonly SOURCE_REVISION_FILE="${BUILD_ROOT}/source-revision"
+
+# The Isaac Sim launcher owns a Python installation whose console-script
+# directory is not always on ``PATH`` in a test container.  CMake finds the
+# same ``uv`` executable that the launcher installs only after this export.
+export PATH="${ISAACLAB_PYTHON_SCRIPTS}:${PATH}"
+
+if ! command -v git >/dev/null || ! dpkg-query --show --showformat='${db:Status-Status}' libx11-dev 2>/dev/null | grep -qx installed; then
+    # IsaacTeleop's CMake dependencies are fetched through Git and its static
+    # OpenXR loader selects the Xlib backend.  The minimal Isaac Sim runtime
+    # image used by the test action includes neither build prerequisite.
+    apt-get update
+    DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends git libx11-dev
+fi
+
+if ! command -v uv >/dev/null; then
+    "${ISAACLAB_PYTHON}" -m pip install uv
+fi
+
+if [[ ! -f "${SOURCE_REVISION_FILE}" ]] \
+    || [[ "$(<"${SOURCE_REVISION_FILE}")" != "${ISAAC_TELEOP_PR769_MERGE_SHA}" ]]; then
+    rm -rf "${SOURCE_DIR}" "${CMAKE_BUILD_DIR}"
+    mkdir -p "${BUILD_ROOT}"
+    if command -v git >/dev/null; then
+        git init --quiet "${SOURCE_DIR}"
+        git -C "${SOURCE_DIR}" remote add origin "${ISAAC_TELEOP_REPOSITORY}"
+        git -C "${SOURCE_DIR}" fetch --depth 1 origin "${ISAAC_TELEOP_PR769_MERGE_SHA}"
+        git -C "${SOURCE_DIR}" checkout --quiet --detach FETCH_HEAD
+        test "$(git -C "${SOURCE_DIR}" rev-parse HEAD)" = "${ISAAC_TELEOP_PR769_MERGE_SHA}"
+    else
+        # Isaac Sim runtime images do not all include Git.  GitHub's archive
+        # endpoint names the immutable commit directly, so this fallback pins
+        # the identical merge revision rather than installing a branch tip.
+        archive_path="$(mktemp "${BUILD_ROOT}/isaacteleop-${ISAAC_TELEOP_PR769_MERGE_SHA}.XXXXXX.tar.gz")"
+        curl --fail --location --retry 3 --retry-delay 2 \
+            "https://github.com/NVIDIA/IsaacTeleop/archive/${ISAAC_TELEOP_PR769_MERGE_SHA}.tar.gz" \
+            --output "${archive_path}"
+        mkdir -p "${SOURCE_DIR}"
+        tar --extract --gzip --file "${archive_path}" --strip-components=1 --directory "${SOURCE_DIR}"
+        rm -f "${archive_path}"
+    fi
+    printf '%s\n' "${ISAAC_TELEOP_PR769_MERGE_SHA}" > "${SOURCE_REVISION_FILE}"
+fi
+
+test -f "${SOURCE_DIR}/CMakeLists.txt"
+test "$(<"${SOURCE_REVISION_FILE}")" = "${ISAAC_TELEOP_PR769_MERGE_SHA}"
+
+cmake -S "${SOURCE_DIR}" -B "${CMAKE_BUILD_DIR}" \
+    -DBUILD_EXAMPLES=OFF \
+    -DBUILD_PLUGINS=OFF \
+    -DBUILD_TESTING=OFF \
+    -DBUILD_VIZ=OFF \
+    -DENABLE_CLANG_FORMAT_CHECK=OFF \
+    -DCMAKE_BUILD_TYPE=Release
+cmake --build "${CMAKE_BUILD_DIR}" --target python_wheel --parallel
+
+wheel_path="$(find "${CMAKE_BUILD_DIR}/wheels" -maxdepth 1 -name 'isaacteleop-*.whl' -print -quit)"
+test -n "${wheel_path}"
+"${ISAACLAB_PYTHON}" -m pip install --force-reinstall --no-deps "${wheel_path}"
+"${ISAACLAB_PYTHON}" -c 'from isaacteleop.retargeters.DVRK.control import DVRKPSMCartesianClutchStateMachine'

scripts/tools/preflight_dvrk_needle_pass_assets.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Verify the remote assets pinned by the dVRK needle-pass task."""
+
+from pathlib import Path
+from runpy import run_path
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
+needle_assets = run_path(
+    REPOSITORY_ROOT / "source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/needle_pass/assets.py"
+)
+dvrk_asset = run_path(REPOSITORY_ROOT / "source/isaaclab_assets/isaaclab_assets/robots/dvrk_asset.py")
+
+NEEDLE_ASSET = needle_assets["NEEDLE_ASSET"]
+SUTURE_PAD_ASSET = needle_assets["SUTURE_PAD_ASSET"]
+verify_remote_asset_sha256 = needle_assets["verify_remote_asset_sha256"]
+DVRK_PSM_USD_PATH = dvrk_asset["DVRK_PSM_USD_PATH"]
+DVRK_PSM_USD_SHA256 = dvrk_asset["DVRK_PSM_USD_SHA256"]
+
+
+def main() -> None:
+    """Download and hash each task asset as an explicit online preflight."""
+
+    assets = (
+        ("needle", NEEDLE_ASSET.url, NEEDLE_ASSET.sha256),
+        ("suture pad", SUTURE_PAD_ASSET.url, SUTURE_PAD_ASSET.sha256),
+        ("dVRK PSM", DVRK_PSM_USD_PATH, DVRK_PSM_USD_SHA256),
+    )
+    for name, url, sha256 in assets:
+        verify_remote_asset_sha256(url, sha256)
+        print(f"Verified {name}: {sha256}")
+
+
+if __name__ == "__main__":
+    main()

source/isaaclab/config/extension.toml

@@ -1,7 +1,7 @@
 [package]
 
 # Note: Semantic Versioning is used: https://semver.org/
-version = "0.54.4"
+version = "0.54.5"
 
 # Description
 title = "Isaac Lab framework for Robot Learning"