adapt for dexterous hands

benchmark.yaml

@@ -1,143 +1,204 @@
 general:
-  scan_project: true
   root_paths:
-    asset: /home/zhiyuan/zhujuan/joysim/gen_data/data    # Root directory for assets (robots, objects, scene USDs, etc.)
+    asset: /home/zhiyuan/zhujuan/joysim_exp/gen_data/data    # Root directory for assets (robots, objects, scene USDs, etc.)
     checkpoints: /home/zhiyuan/zhujuan/checkpoints
-    output: /home/zhiyuan/zhujuan/joysim/output                   # Root directory for outputs (recorded data, logs, etc.)
+    output: /home/zhiyuan/zhujuan/joysim_exp/output                   # Root directory for outputs (recorded data, logs, etc.)
 
 simulation:
   stereotype: isaaclab
-  intiailize_steps: 300
   launch_config:
     device: cuda
     enable_cameras: true
     headless: false
     livestream: 0
-    
-    
 scene:
-  name: 827313_home
+  name: kujiale_multispace
   base_config:
     stereotype: usd
-    name: _827313_home_workspace_01
+    name: _827313_home_workspace_00
     source: platform
-    asset_path: platform://scenes/kujiale_multispace/827313_home/collect_asset_without_phy.optimized.glb
+    asset_path: platform://scenes/kujiale_multispace/827313_home/workspace_00.usd
   object_cfg_dict:
-    omni6DPose_timer_017:
-      name: omni6DPose_timer_017
+    omni6DPose_can_016:
+      name: omni6DPose_can_016
       stereotype: rigid
-      source: platform
-      asset_path: platform://objects/omni6DPose/timer/omni6DPose_timer_017/Aligned.usd
+      source: local
+      asset_path: asset://objects/omni6DPose/can/omni6DPose_can_016/Aligned.usd
       scale:
       - 0.001
       - 0.001
       - 0.001
       position:
-      - 0.552364
-      - -4.0582599999999995
-      - 0.524713118
-      quaternion:
-      - 0.166210542394157
-      - 0.166210542394157
-      - 0.6872947370648492
-      - 0.6872947370648491
-      axis_y_up: true
+      - 0.419859
+      - -4.02430000000001
+      - 0.510259093
+      rotation:
+      - -0.304408012043137
+      - -0.304408012043137
+      - 0.638228612805745
+      - 0.6382286128057448
     omni6DPose_book_031:
       name: omni6DPose_book_031
       stereotype: rigid
-      source: platform
-      asset_path: platform://objects/omni6DPose/book/omni6DPose_book_031/Aligned.usd
+      source: local
+      asset_path: asset://objects/omni6DPose/book/omni6DPose_book_031/Aligned.usd
       scale:
       - 0.001
       - 0.001
       - 0.001
       position:
-      - 0.6623640000000001
-      - -3.7882599999999997
-      - 0.5101601435
-      quaternion:
-      - 0.7063055546421202
-      - 0.7063055546421203
-      - -0.03365209475927027
-      - -0.033652094759270265
+      - 0.419859
+      - -4.152430000000001
+      - 0.510259093
+      quaternion: [1, 0, 0, 0]
       axis_y_up: true
+
   robot_cfg_dict:
     r1pro_dex:
       name: r1pro_dex
       asset_path: asset://robots/r1pro/r1pro_dex.usd
-      position:
-      - 1.082364
-      - -3.92826
-      - 0.47629299999999997
-      rotation: 
-      - 7.549799991308018e-08
-      - 0.0
-      - 0.0
-      - 0.9999999999999973
+      position: [-0.5, -4.0, 0.0]
+      rotation: [1, 0, 0, 0]
       stereotype: modular_robot
       source: local
-      arm_modules:
-        main_arm:
-          arm_actuator_name: dex_arm
-          ee_actuator_name: robot_hand
-          ee_type: dexterous_hand
+
+      init_joint_position:
+        torso_joint1: 0.0
+        torso_joint2: 0.0
+        torso_joint3: 0.0
+        torso_joint4: 0.0
+        left_arm_joint1: 0.0
+        left_arm_joint2: 0.5
+        left_arm_joint3: 0.0
+        left_arm_joint4: -1.0
+        left_arm_joint5: 0.0
+        left_arm_joint6: 0.0
+        left_arm_joint7: 0.0
+        right_arm_joint1: 0.0
+        right_arm_joint2: -0.5
+        right_arm_joint3: 0.0
+        right_arm_joint4: -1.0
+        right_arm_joint5: 0.0
+        right_arm_joint6: 0.0
+        right_arm_joint7: 0.0
+
       actuator_cfg_dict:
-        dex_arm:
+        left_arm:
           stereotype: arm
-          joint_names_expr: [
-              left_arm_joint1, right_arm_joint1, left_arm_joint2, right_arm_joint2,
-              left_arm_joint3, right_arm_joint3, left_arm_joint4, right_arm_joint4,
-              left_arm_joint5, right_arm_joint5, left_arm_joint6, right_arm_joint6,
-              left_arm_joint7, right_arm_joint7]
-          stiffness: 3000.0
-          damping: 800.0
-        robot_hand:
-          stereotype: dexterous_hand
-          joint_names_expr: [ 
-              left_index_MCP_FE, left_middle_MCP_FE, left_pinky_CMC, left_ring_MCP_FE, left_thumb_CMC_FE,
-              right_index_MCP_FE, right_middle_MCP_FE, right_pinky_CMC, right_ring_MCP_FE, right_thumb_CMC_FE,
-              left_index_MCP_AA, left_middle_MCP_AA, left_pinky_MCP_FE, left_ring_MCP_AA, left_thumb_CMC_AA,
-              right_index_MCP_AA, right_middle_MCP_AA, right_pinky_MCP_FE, right_ring_MCP_AA, right_thumb_CMC_AA,
-              left_index_PIP, left_middle_PIP, left_pinky_MCP_AA, left_ring_PIP, left_thumb_MCP_FE,
-              right_index_PIP, right_middle_PIP, right_pinky_MCP_AA, right_ring_PIP, right_thumb_MCP_FE,
-              left_index_DIP, left_middle_DIP, left_pinky_PIP, left_ring_DIP, left_thumb_MCP_AA,
-              right_index_DIP, right_middle_DIP, right_pinky_PIP, right_ring_DIP, right_thumb_MCP_AA,
-              left_pinky_DIP, left_thumb_IP, right_pinky_DIP, right_thumb_IP]
-          stiffness: 10000
-          damping: 500.0
-          close_control_type: velocity
-          open_control_type: position
+          joint_names_expr: [left_arm_joint1, left_arm_joint2, left_arm_joint3, left_arm_joint4, left_arm_joint5, left_arm_joint6, left_arm_joint7]
+          stiffness: 60000.0
+          damping: 4000.0
+        right_arm:
+          stereotype: arm
+          joint_names_expr: [right_arm_joint1, right_arm_joint2, right_arm_joint3, right_arm_joint4, right_arm_joint5, right_arm_joint6, right_arm_joint7]
+          stiffness: 60000.0
+          damping: 4000.0
+        left_hand:
+          stereotype: arm
+          joint_names_expr: [left_thumb_CMC_FE, left_thumb_CMC_AA, left_thumb_MCP_FE, left_thumb_MCP_AA, left_thumb_IP, left_index_MCP_FE, left_index_MCP_AA, left_index_PIP, left_index_DIP, left_middle_MCP_FE, left_middle_MCP_AA, left_middle_PIP, left_middle_DIP, left_ring_MCP_FE, left_ring_MCP_AA, left_ring_PIP, left_ring_DIP, left_pinky_CMC, left_pinky_MCP_FE, left_pinky_MCP_AA, left_pinky_PIP, left_pinky_DIP]
+          stiffness: 50.0
+          damping: 5.0
+        right_hand:
+          stereotype: arm
+          joint_names_expr: [right_thumb_CMC_FE, right_thumb_CMC_AA, right_thumb_MCP_FE, right_thumb_MCP_AA, right_thumb_IP, right_index_MCP_FE, right_index_MCP_AA, right_index_PIP, right_index_DIP, right_middle_MCP_FE, right_middle_MCP_AA, right_middle_PIP, right_middle_DIP, right_ring_MCP_FE, right_ring_MCP_AA, right_ring_PIP, right_ring_DIP, right_pinky_CMC, right_pinky_MCP_FE, right_pinky_MCP_AA, right_pinky_PIP, right_pinky_DIP]
+          stiffness: 50.0
+          damping: 5.0
+        torso:
+          stereotype: arm
+          joint_names_expr: [torso_joint1, torso_joint2, torso_joint3, torso_joint4]
+          stiffness: 100000.0
+          damping: 8000.0
+        base_lock:
+          stereotype: arm
+          joint_names_expr: [steer_motor_joint1, steer_motor_joint2, steer_motor_joint3, wheel_motor_joint1, wheel_motor_joint2, wheel_motor_joint3]
+          stiffness: 100000.0
+          damping: 5000.0
+
+      arm_modules:
+        left_arm:
+          arm_actuator_name: left_arm
+          ee_link_name: left_hand_C_MC
+          ee_type: dexterous_hand
+          ee_actuator_name: left_hand
+        right_arm:
+          arm_actuator_name: right_arm
+          ee_link_name: right_hand_C_MC
+          ee_type: dexterous_hand
+          ee_actuator_name: right_hand
+
+      extra_modules:
+        torso:
+          actuator_name: torso
+
       use_planner: false
+
   sensor_cfg_dict:
-    Zed_Camera:
-      name: Zed_Camera
+
+    head_camera:
+      name: head_camera
       stereotype: camera
-      data_types:
-      - rgb
-      - depth
-      - normals
+      position: [-0.4, -4.0, 1.2]
+      look_at:
+        is_point: true
+        look_at_point: [0.4200, -4.1530, 0.4885]
+      data_types: [rgb]
+      width: 1280
+      height: 720
+      camera_model: pinhole
+      fix_camera: true
+    front_camera:
+      name: front_camera
+      stereotype: camera
+      position: [1.0, -4.0, 1.5]
+      look_at:
+        is_point: true
+        look_at_point: [-1.0, -4.0, 1.2]
+      data_types: [rgb]
+      width: 1280
+      height: 720
+      camera_model: pinhole
+      fix_camera: true
+    left_camera:
+      name: left_camera
+      stereotype: camera
+      position: [-1.0, -1.0, 1.2]
+      look_at:
+        is_point: true
+        look_at_point: [0.0, -4.1, 1.2]
+      data_types: [rgb]
+      width: 1280
+      height: 720
+      camera_model: pinhole
+      fix_camera: true
+    right_camera:
+      name: right_camera
+      stereotype: camera
+      position: [-1.0, -6.5, 1.2]
+      look_at:
+        is_point: true
+        look_at_point: [0.0, -4.1, 1.2]
+      data_types: [rgb]
       width: 1280
       height: 720
       camera_model: pinhole
       fix_camera: true
-      focal_length: 2.8
-      horizontal_aperture: 4.893416860031241
-      vertical_aperture: 2.7608816125932627
-      convention: opengl
-      attach_to:
-        target_name: r1pro_dex
-        is_articulation_part: false
-        create_fixed_joint: true
-        local_position:
-        - 0.06
-        - 0.0
-        - 0.01
-        local_rotation:
-        - -1.0
-        - 0.0
-        - 0.0
-        - 0.0
 
+  light_cfg_dict:
+    sun:
+      name: sun
+      stereotype: general_light
+      light_type: distant
+      position: [0, 0, 5]
+      rotation: [1, 0, 0, 0]
+      intensity: 1000
+      angle: 0.53
+      color: [1.0, 1.0, 1.0]
+    sky:
+      name: sky
+      stereotype: general_light
+      light_type: dome
+      intensity: 10.0
+      color: [1.0, 1.0, 1.0]
 extension:
   extension_cfg_dict:
     benchmark_data_collect:
@@ -146,21 +207,51 @@ extension:
       observer_cfgs:
         - stereotype: robot_observer
           name: r1pro_dex
-          target_joint_names: [steer_motor_joint1, steer_motor_joint2, steer_motor_joint3, torso_joint1, wheel_motor_joint1,
-              wheel_motor_joint2, wheel_motor_joint3, torso_joint2, torso_joint3, torso_joint4,
-              left_arm_joint1, right_arm_joint1, left_arm_joint2, right_arm_joint2,
-              left_arm_joint3, right_arm_joint3, left_arm_joint4, right_arm_joint4,
-              left_arm_joint5, right_arm_joint5, left_arm_joint6, right_arm_joint6,
-              left_arm_joint7, right_arm_joint7,
-              left_index_MCP_FE, left_middle_MCP_FE, left_pinky_CMC, left_ring_MCP_FE, left_thumb_CMC_FE,
-              right_index_MCP_FE, right_middle_MCP_FE, right_pinky_CMC, right_ring_MCP_FE, right_thumb_CMC_FE,
-              left_index_MCP_AA, left_middle_MCP_AA, left_pinky_MCP_FE, left_ring_MCP_AA, left_thumb_CMC_AA,
-              right_index_MCP_AA, right_middle_MCP_AA, right_pinky_MCP_FE, right_ring_MCP_AA, right_thumb_CMC_AA,
-              left_index_PIP, left_middle_PIP, left_pinky_MCP_AA, left_ring_PIP, left_thumb_MCP_FE,
-              right_index_PIP, right_middle_PIP, right_pinky_MCP_AA, right_ring_PIP, right_thumb_MCP_FE,
-              left_index_DIP, left_middle_DIP, left_pinky_PIP, left_ring_DIP, left_thumb_MCP_AA,
-              right_index_DIP, right_middle_DIP, right_pinky_PIP, right_ring_DIP, right_thumb_MCP_AA,
-              left_pinky_DIP, left_thumb_IP, right_pinky_DIP, right_thumb_IP]
+          target_joint_names:
+              - left_thumb_CMC_FE
+              - left_thumb_CMC_AA
+              - left_thumb_MCP_FE
+              - left_thumb_MCP_AA
+              - left_thumb_IP
+              - left_index_MCP_FE
+              - left_index_MCP_AA
+              - left_index_PIP
+              - left_index_DIP
+              - left_middle_MCP_FE
+              - left_middle_MCP_AA
+              - left_middle_PIP
+              - left_middle_DIP
+              - left_ring_MCP_FE
+              - left_ring_MCP_AA
+              - left_ring_PIP
+              - left_ring_DIP
+              - left_pinky_CMC
+              - left_pinky_MCP_FE
+              - left_pinky_MCP_AA
+              - left_pinky_PIP
+              - left_pinky_DIP
+              - right_thumb_CMC_FE
+              - right_thumb_CMC_AA
+              - right_thumb_MCP_FE
+              - right_thumb_MCP_AA
+              - right_thumb_IP
+              - right_index_MCP_FE
+              - right_index_MCP_AA
+              - right_index_PIP
+              - right_index_DIP
+              - right_middle_MCP_FE
+              - right_middle_MCP_AA
+              - right_middle_PIP
+              - right_middle_DIP
+              - right_ring_MCP_FE
+              - right_ring_MCP_AA
+              - right_ring_PIP
+              - right_ring_DIP
+              - right_pinky_CMC
+              - right_pinky_MCP_FE
+              - right_pinky_MCP_AA
+              - right_pinky_PIP
+              - right_pinky_DIP
           observe_ee_pose: true
           observe_ee_state: true
           observe_joint_position: true
@@ -171,7 +262,16 @@ extension:
           observe_joint_position_targets: true
           observe_joint_velocity_targets: true
         - stereotype: sensor_observer
-          name: Zed_Camera
+          name: head_camera
+          observe_rgb: true
+        - stereotype: sensor_observer
+          name: front_camera
+          observe_rgb: true
+        - stereotype: sensor_observer
+          name: left_camera
+          observe_rgb: true
+        - stereotype: sensor_observer
+          name: right_camera
           observe_rgb: true
           
     starvla_benchmark:
@@ -181,17 +281,17 @@ extension:
       action_frequency: 15.0
       timeout_per_episode: 300
       goals:
-        - name: cola on top of book
-          description: check if the cola bottle is on the book
+        - name: can on top of book
+          description: check if the can is on the book
           stereotype: on_top
           object_A_name: omni6DPose_book_031
-          object_B_name: omni6DPose_timer_017
+          object_B_name: omni6DPose_can_016
       policy:
         stereotype: starvla
         robot_name: r1pro_dex
-        arm_name: main_arm
-        sensor_names: [Zed_Camera]
-        prompt: pick up the timer and put on the book
+        arm_name: right_arm
+        sensor_names: [head_camera]
+        prompt: pick up the can and put on the book
         run_trunk_size: 16
         visualize_action_ee_pose: true
         visualize_state_ee_pose: true
@@ -203,7 +303,7 @@ extension:
       data_collector_name: benchmark_data_collect
       record_fps: 30
       backend_root_path: output://benchmark_record
-      postprocess_list: ["hdf5", "video"]
+      postprocess_list: ["hdf5", "video", "preview_video"]
 
 policy_server:
   ckpt_path: checkpoints://egodex_part1_restats_gbs1024/checkpoints/steps_70000_pytorch_model.pt
@@ -211,5 +311,3 @@ policy_server:
   host: 0.0.0.0
   port: 5000
   use_bf16: true
-  unnorm_key: oxe_bridge
-  state_mode: ee_pose7