import newton
import newton.ik as ik
import numpy as np
import warp as wp
from lwmr.utils import _config, create_viewer_viser, enable_target_control, set_initial_state
from tqdm.auto import tqdm
wp.config.quiet = TrueAsset Import
In this notebook, we will cover how to import assets into a Newton simulation.
This example is taken from NVIDIA’s GTC 2026 presentation Accelerate Robot Learning With NVIDIA Isaac Lab and Newton.
FRAME_STEP = 1.0 / 60.0
SIM_SUBSTEPS = 10
TIME_STEP = FRAME_STEP / SIM_SUBSTEPSdef build_world():
builder = newton.ModelBuilder()
builder.default_shape_cfg.gap = 0.0
newton.solvers.SolverMuJoCo.register_custom_attributes(builder)
# Add a table
table_height = 0.1
table_position = wp.vec3(0.0, -0.5, 0.5 * table_height)
builder.add_shape_box(body=-1, hx=0.4, hy=0.4, hz=0.5 * table_height, xform=wp.transform(table_position))
# Add the Franka Emika Panda robot
table_top_center = table_position + wp.vec3(0.0, 0.0, 0.5 * table_height)
robot_base_position = table_top_center + wp.vec3(-0.5, 0.0, 0.0)
builder.add_urdf(
str(newton.utils.download_asset("franka_emika_panda") / "urdf/fr3_franka_hand.urdf"),
xform=wp.transform(robot_base_position, wp.quat_identity()), # type: ignore
floating=False,
enable_self_collisions=False,
parse_visuals_as_colliders=False,
)
set_initial_state(builder)
enable_target_control(builder)
# Add a cube to manipulate
cube_size = 0.05
cube_half_size = 0.05 * 0.5
cube_body = None
cube_position = table_top_center + wp.vec3(0.0, 0.15, cube_half_size)
cube_body = builder.add_body(xform=wp.transform(cube_position, wp.quat_identity())) # type: ignore
shape_cfg = newton.ModelBuilder.ShapeConfig(margin=1e-3, density=400.0)
builder.add_shape_box(
body=cube_body, hx=cube_half_size, hy=cube_half_size, hz=cube_half_size, cfg=shape_cfg
)
return builder, cube_size, table_position, cube_bodybuilder, size, position, body = build_world()
model = builder.finalize()
state_0 = model.state()
state_1 = model.state()
control = model.control()
pipeline = newton.CollisionPipeline(model)
contacts = pipeline.contacts()
newton.eval_fk(model, model.joint_q, model.joint_qd, state_0) # type: ignoreWarp 1.18.0 initialized:
CUDA not enabled in this build
Devices:
"cpu" : "arm"
Kernel cache:
~/Library/Caches/warp/1.18.0
Module newton._src.geometry.inertia f6b4bd8 load on device 'cpu' took 20.72 ms (cached)
Module newton._src.utils.mesh 92b663b load on device 'cpu' took 15.53 ms (cached)
Module validate_and_correct_inertia_kernel_1ff2ac62 6ad45b8 load on device 'cpu' took 4.48 ms (cached)
Module newton._src.geometry.bvh 5fa2076 load on device 'cpu' took 1.11 ms (cached)
Module newton._src.sim.articulation 1e9d119 load on device 'cpu' took 1.51 ms (cached)solver = newton.solvers.SolverMuJoCo(
model,
solver="newton",
integrator="implicitfast",
iterations=20,
ls_iterations=100,
nconmax=1000,
njmax=2000,
cone="elliptic",
impratio=1000.0,
use_mujoco_contacts=False,
)Module newton._src.solvers.mujoco.kernels c5847ab load on device 'cpu' took 1.32 ms (cached)
Module mujoco_warp._src.io 593bb82 load on device 'cpu' took 1.11 ms (cached)
Module mujoco_warp._src.smooth 5a7824c load on device 'cpu' took 1.19 ms (cached)
Module _small_cholesky_factorize_block__locals__kernel_79a57364 9adc210 load on device 'cpu' took 1.02 ms (cached)
Module _tile_cholesky_factorize_block__locals__kernel_f101fbbf 4a780fc load on device 'cpu' took 21.07 ms (cached)
Module _small_cholesky_solve_block__locals__kernel_c4e2dcc2 6e86cbf load on device 'cpu' took 0.87 ms (cached)
Module _tile_cholesky_solve_block__locals__kernel_818a5faf 05690a2 load on device 'cpu' took 13.21 ms (cached)ee_index = 11
fixed_rot = wp.quat_from_axis_angle(wp.vec3(1.0, 0.0, 0.0), wp.pi) # type: ignore
home_pos_np = state_0.body_q.numpy()[ee_index][:3].astype(np.float32) # type: ignore
home_pos = wp.vec3(home_pos_np[0], home_pos_np[1], home_pos_np[2])
pos_obj = ik.IKObjectivePosition(
link_index=ee_index,
link_offset=wp.vec3(0.0, 0.0, 0.0),
target_positions=wp.array([home_pos], dtype=wp.vec3),
)
rot_obj = ik.IKObjectiveRotation(
link_index=ee_index,
link_offset_rotation=wp.quat_identity(), # type: ignore
target_rotations=wp.array([fixed_rot[:4]], dtype=wp.vec4), # type: ignore
)
joint_limit_obj = ik.IKObjectiveJointLimit(
joint_limit_lower=model.joint_limit_lower, # type: ignore
joint_limit_upper=model.joint_limit_upper, # type: ignore
)
joint_q_ik = wp.clone(model.joint_q.reshape((1, -1))) # type: ignore
ik_solver = ik.IKSolver(
model=model,
n_problems=1,
objectives=[pos_obj, rot_obj, joint_limit_obj],
lambda_initial=0.1,
jacobian_mode=ik.IKJacobianType.ANALYTIC,
)cube_pos_np = state_0.body_q.numpy()[body][:3] # type: ignore
cube_position = wp.vec3(cube_pos_np[0], cube_pos_np[1], cube_pos_np[2])
approach_offset = wp.vec3(0.0, 0.0, 3.0 * size)
lift_offset = wp.vec3(0.0, 0.0, 4.0 * size)
height = 0.1
center = position + wp.vec3(0.0, 0.0, 0.5 * height)
drop_pos = center + wp.vec3(0.0, -0.15, 0.5 * size)
grip_open = _config.FINGER_OPEN
grip_closed = _config.FINGER_CLOSED
sequence = [
{"name": "approach", "pos": cube_position + approach_offset, "grip": grip_open, "frames": 60},
{"name": "descend", "pos": cube_position, "grip": grip_open, "frames": 60},
{"name": "close", "pos": cube_position, "grip": (grip_open, grip_closed), "frames": 60},
{"name": "lift", "pos": cube_position + lift_offset, "grip": grip_closed, "frames": 60},
{"name": "move", "pos": drop_pos + approach_offset, "grip": grip_closed, "frames": 60},
{"name": "place", "pos": drop_pos, "grip": grip_closed, "frames": 60},
{"name": "hold", "pos": drop_pos, "grip": grip_closed, "frames": 30},
{"name": "open", "pos": drop_pos, "grip": (grip_closed, grip_open), "frames": 45},
{"name": "retract", "pos": drop_pos + approach_offset, "grip": grip_open, "frames": 60},
]def run_sim_substeps():
global state_0, state_1
for _ in range(SIM_SUBSTEPS):
state_0.clear_forces()
pipeline.collide(state_0, contacts)
solver.step(state_in=state_0, state_out=state_1, control=control, contacts=contacts, dt=TIME_STEP)
state_0, state_1 = state_1, state_0viewer = create_viewer_viser("franka", model, quiet=True, overwrite=False)
viewer.set_model(model)
viewer.set_camera(wp.vec3(0.5, 0.0, 0.5), -15, -140)
IK_ITERS = 24
ARM_DOFS = _config.ARM_DOF_COUNT
DOFS = _config.DOF_COUNT
sim_time = 0.0
graph = None
if wp.get_device().is_cuda:
with wp.ScopedCapture() as capture:
run_sim_substeps()
graph = capture.graph
phase_bar = tqdm(sequence, desc="phase: init")
for phase in phase_bar:
phase_bar.set_description(f"phase: {phase['name']}")
target_pos = phase["pos"]
frames = int(phase["frames"])
grip = phase["grip"]
if isinstance(grip, tuple):
grip_start, grip_end = float(grip[0]), float(grip[1])
else:
grip_start = grip_end = float(grip)
for frame in range(frames):
alpha = (frame + 1) / frames if frames > 1 else 1.0
grip_i = grip_start + (grip_end - grip_start) * alpha
pos_obj.set_target_positions(wp.array([target_pos], dtype=wp.vec3))
rot_obj.set_target_rotations(wp.array([fixed_rot[:4]], dtype=wp.vec4)) # type: ignore
ik_solver.step(joint_q_ik, joint_q_ik, iterations=IK_ITERS) # type: ignore
joint_target_pos_view = control.joint_target_q.reshape((1, -1)) # type: ignore
wp.copy(dest=joint_target_pos_view[:, :ARM_DOFS], src=joint_q_ik[:, :ARM_DOFS]) # type: ignore
wp.copy(
dest=joint_target_pos_view[:, ARM_DOFS:DOFS], src=wp.array([[grip_i, grip_i]], dtype=wp.float32)
) # type: ignore
if graph is not None:
wp.capture_launch(graph)
else:
run_sim_substeps()
viewer.begin_frame(sim_time)
viewer.log_state(state_0)
viewer.end_frame()
sim_time += FRAME_STEP
viewer.show_notebook()Module newton._src.viewer.kernels 3942d1f load on device 'cpu' took 2.46 ms (cached)
phase: init: 0%| | 0/9 [00:00<?, ?it/s]
phase: approach: 0%| | 0/9 [00:00<?, ?it/s]Module newton._src.sim.ik.ik_objectives e76812b load on device 'cpu' took 13.59 ms (cached)
Module newton._src.sim.ik.ik_solver 9ae4091 load on device 'cpu' took 12.07 ms (cached)
Module IKOptimizerLM___build_specialized__locals___fk_local_9430eeda 9d4700b load on device 'cpu' took 7.36 ms (cached)
Module newton._src.sim.ik.ik_common 6de7874 load on device 'cpu' took 6.52 ms (cached)
Module IKOptimizerLM___build_specialized__locals___compute_motion_subspace_2d_c5d26616 94a31f2 load on device 'cpu' took 6.18 ms (cached)
Module _lm_solve_tiled_15_21_ff235b1e b36d56c load on device 'cpu' took 7.03 ms (cached)
Module newton._src.sim.ik.ik_lm_optimizer 43dab9e load on device 'cpu' took 6.50 ms (cached)
Module newton._src.sim.collide a0e9da6 load on device 'cpu' took 0.77 ms (cached)
Module newton._src.geometry.broad_phase_nxn 4413549 load on device 'cpu' took 0.63 ms (cached)
Module narrow_phase_primitive_write_contact 066b49c load on device 'cpu' took 0.84 ms (cached)
Module narrow_phase_gjk_mpr_True_write_contact_support_map_lean_post_process_minkowski_only 2915b37 load on device 'cpu' took 1.06 ms (cached)
Module newton._src.geometry.narrow_phase 72cccae load on device 'cpu' took 0.85 ms (cached)
Module newton._src.geometry.contact_reduction_global de5b807 load on device 'cpu' took 9.63 ms (cached)
Module newton._src.geometry.sdf_contact 7975f62 load on device 'cpu' took 7.07 ms (cached)
Module narrow_phase_mesh_plane_write_contact_to_reducer_True 4127b8c load on device 'cpu' took 6.80 ms (cached)
Module mesh_triangle_contacts_to_reducer_kernel_e41ecfb6 6416904 load on device 'cpu' took 7.77 ms (cached)
Module sdf_contact_write_contact_to_reducer_False_True 02c6cf4 load on device 'cpu' took 10.47 ms (cached)
Module export_reduced_contacts_write_contact 5207f54 load on device 'cpu' took 7.26 ms (cached)
Module mujoco_warp._src.constraint 90a64f5 load on device 'cpu' took 0.84 ms (cached)
Module _equality_joint__locals__kernel_66ba6c83 f4798ab load on device 'cpu' took 7.52 ms (cached)
Module _friction_dof__locals__kernel_eadd710f ffecde8 load on device 'cpu' took 0.71 ms (cached)
Module _limit_slide_hinge__locals__kernel_cbfc119c 92f4a0e load on device 'cpu' took 5.26 ms (cached)
Module _efc_contact_init__locals__kernel_48b1da0a f0bc1a0 load on device 'cpu' took 6.78 ms (cached)
Module _efc_contact_jac_dense__locals__kernel_cde1aa89 90af167 load on device 'cpu' took 8.00 ms (cached)
Module _efc_contact_update__locals__kernel_cf1f607c 3637bfa load on device 'cpu' took 8.37 ms (cached)
Module mujoco_warp._src.forward b904ce8 load on device 'cpu' took 0.86 ms (cached)
Module mujoco_warp._src.passive e5677c9 load on device 'cpu' took 0.72 ms (cached)
Module _qfrc_smooth__locals__kernel_8b823980 59d943b load on device 'cpu' took 0.67 ms (cached)
Module mujoco_warp._src.support 79faa31 load on device 'cpu' took 0.66 ms (cached)
Module _small_cholesky_factorize_solve_block__locals__kernel_9da335f1 f1514dc load on device 'cpu' took 0.69 ms (cached)
Module _tile_cholesky_factorize_solve_block__locals__kernel_5c841c33 2aa8ce5 load on device 'cpu' took 7.17 ms (cached)
Module _solve_init_dof__locals__kernel_1d7ecfa4 b626280 load on device 'cpu' took 0.76 ms (cached)
Module mujoco_warp._src.solver e441f3d load on device 'cpu' took 0.75 ms (cached)
Module _solve_init_jaref_kernel__locals__kernel_2edc21a2 b6ce6e7 load on device 'cpu' took 6.85 ms (cached)
Module mul_m_kernel__locals___mul_m_d74bd8d9 ec4adf4 load on device 'cpu' took 0.80 ms (cached)
Module _update_constraint_efc__locals__kernel_4b172e8b 1b383ee load on device 'cpu' took 0.70 ms (cached)
Module _update_constraint_init_qfrc_constraint_dense__locals__kernel_c29c2cfc 711ed07 load on device 'cpu' took 0.74 ms (cached)
Module _update_gradient_zero_grad_dot__locals__kernel_93442399 d6cac50 load on device 'cpu' took 0.64 ms (cached)
Module _update_gradient_grad__locals__kernel_378e7517 71dbe85 load on device 'cpu' took 0.58 ms (cached)
Module _update_gradient_JTDAJ_dense_tiled__locals__kernel_b035a486 a8dfb0c load on device 'cpu' took 8.31 ms (cached)
Module _update_gradient_cholesky__locals__kernel_d33cd7e5 3c9e71c load on device 'cpu' took 7.73 ms (cached)
Module _linesearch_iterative_kernel__locals__kernel_67f5f23e 6979d9f load on device 'cpu' took 6.53 ms (cached)
Module mujoco_warp._src.sensor 8b0053e load on device 'cpu' took 1.08 ms (cached)
Module mujoco_warp._src.derivative 65e089c load on device 'cpu' took 0.79 ms (cached)
Module _next_time_builder__locals___next_time_f604e0b3 324f4ea load on device 'cpu' took 0.80 ms (cached)
phase: approach: 11%|█ | 1/9 [00:02<00:22, 2.85s/it]
phase: descend: 11%|█ | 1/9 [00:02<00:22, 2.85s/it]
phase: descend: 22%|██▏ | 2/9 [00:05<00:18, 2.69s/it]
phase: close: 22%|██▏ | 2/9 [00:05<00:18, 2.69s/it]
phase: close: 33%|███▎ | 3/9 [00:08<00:16, 2.74s/it]
phase: lift: 33%|███▎ | 3/9 [00:08<00:16, 2.74s/it]
phase: lift: 44%|████▍ | 4/9 [00:12<00:16, 3.21s/it]
phase: move: 44%|████▍ | 4/9 [00:12<00:16, 3.21s/it]
phase: move: 56%|█████▌ | 5/9 [00:16<00:13, 3.47s/it]
phase: place: 56%|█████▌ | 5/9 [00:16<00:13, 3.47s/it]
phase: place: 67%|██████▋ | 6/9 [00:19<00:10, 3.37s/it]
phase: hold: 67%|██████▋ | 6/9 [00:19<00:10, 3.37s/it]
phase: hold: 78%|███████▊ | 7/9 [00:20<00:05, 2.64s/it]
phase: open: 78%|███████▊ | 7/9 [00:20<00:05, 2.64s/it]
phase: open: 89%|████████▉ | 8/9 [00:22<00:02, 2.45s/it]
phase: retract: 89%|████████▉ | 8/9 [00:22<00:02, 2.45s/it]
phase: retract: 100%|██████████| 9/9 [00:24<00:00, 2.39s/it]
phase: retract: 100%|██████████| 9/9 [00:24<00:00, 2.75s/it]