Asset Import

In this notebook, we will cover how to import assets into a Newton simulation.

This example is taken from NVIDIA’s GTC 2026 presentation Accelerate Robot Learning With NVIDIA Isaac Lab and Newton.

import newton
import newton.ik as ik
import numpy as np
import warp as wp
from lwmr.utils import _config, create_viewer_viser, enable_target_control, set_initial_state
from tqdm.auto import tqdm

wp.config.quiet = True

/Users/ajcd2020/Documents/Repositories/anthonyjclark/simer-tutorial/2026-icra/.venv/lib/python3.14/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

FRAME_STEP = 1.0 / 60.0
SIM_SUBSTEPS = 10
TIME_STEP = FRAME_STEP / SIM_SUBSTEPS

def build_world():
    builder = newton.ModelBuilder()
    builder.default_shape_cfg.gap = 0.0
    newton.solvers.SolverMuJoCo.register_custom_attributes(builder)

    # Add a table
    table_height = 0.1
    table_position = wp.vec3(0.0, -0.5, 0.5 * table_height)
    builder.add_shape_box(body=-1, hx=0.4, hy=0.4, hz=0.5 * table_height, xform=wp.transform(table_position))

    # Add the Franka Emika Panda robot
    table_top_center = table_position + wp.vec3(0.0, 0.0, 0.5 * table_height)
    robot_base_position = table_top_center + wp.vec3(-0.5, 0.0, 0.0)
    builder.add_urdf(
        str(newton.utils.download_asset("franka_emika_panda") / "urdf/fr3_franka_hand.urdf"),
        xform=wp.transform(robot_base_position, wp.quat_identity()),  # type: ignore
        floating=False,
        enable_self_collisions=False,
        parse_visuals_as_colliders=False,
    )
    set_initial_state(builder)
    enable_target_control(builder)

    # Add a cube to manipulate
    cube_size = 0.05
    cube_half_size = 0.05 * 0.5
    cube_body = None
    cube_position = table_top_center + wp.vec3(0.0, 0.15, cube_half_size)
    cube_body = builder.add_body(xform=wp.transform(cube_position, wp.quat_identity()))  # type: ignore
    shape_cfg = newton.ModelBuilder.ShapeConfig(margin=1e-3, density=400.0)
    builder.add_shape_box(
        body=cube_body, hx=cube_half_size, hy=cube_half_size, hz=cube_half_size, cfg=shape_cfg
    )

    return builder, cube_size, table_position, cube_body

builder, size, position, body = build_world()
model = builder.finalize()

state_0 = model.state()
state_1 = model.state()
control = model.control()
contacts = model.contacts()

newton.eval_fk(model, model.joint_q, model.joint_qd, state_0)  # type: ignore

solver = newton.solvers.SolverMuJoCo(
    model,
    solver="newton",
    integrator="implicitfast",
    iterations=20,
    ls_parallel=True,
    ls_iterations=100,
    nconmax=1000,
    njmax=2000,
    cone="elliptic",
    impratio=1000.0,
    use_mujoco_contacts=False,
)

ee_index = 11

fixed_rot = wp.quat_from_axis_angle(wp.vec3(1.0, 0.0, 0.0), wp.pi)  # type: ignore
home_pos_np = state_0.body_q.numpy()[ee_index][:3].astype(np.float32)  # type: ignore
home_pos = wp.vec3(home_pos_np[0], home_pos_np[1], home_pos_np[2])

pos_obj = ik.IKObjectivePosition(
    link_index=ee_index,
    link_offset=wp.vec3(0.0, 0.0, 0.0),
    target_positions=wp.array([home_pos], dtype=wp.vec3),
)

rot_obj = ik.IKObjectiveRotation(
    link_index=ee_index,
    link_offset_rotation=wp.quat_identity(),  # type: ignore
    target_rotations=wp.array([fixed_rot[:4]], dtype=wp.vec4),  # type: ignore
)

joint_limit_obj = ik.IKObjectiveJointLimit(
    joint_limit_lower=model.joint_limit_lower,  # type: ignore
    joint_limit_upper=model.joint_limit_upper,  # type: ignore
)

joint_q_ik = wp.clone(model.joint_q.reshape((1, -1)))  # type: ignore

ik_solver = ik.IKSolver(
    model=model,
    n_problems=1,
    objectives=[pos_obj, rot_obj, joint_limit_obj],
    lambda_initial=0.1,
    jacobian_mode=ik.IKJacobianType.ANALYTIC,
)

cube_pos_np = state_0.body_q.numpy()[body][:3]  # type: ignore
cube_position = wp.vec3(cube_pos_np[0], cube_pos_np[1], cube_pos_np[2])

approach_offset = wp.vec3(0.0, 0.0, 3.0 * size)
lift_offset = wp.vec3(0.0, 0.0, 4.0 * size)
height = 0.1
center = position + wp.vec3(0.0, 0.0, 0.5 * height)
drop_pos = center + wp.vec3(0.0, -0.15, 0.5 * size)

grip_open = _config.FINGER_OPEN
grip_closed = _config.FINGER_CLOSED

sequence = [
    {"name": "approach", "pos": cube_position + approach_offset, "grip": grip_open, "frames": 60},
    {"name": "descend", "pos": cube_position, "grip": grip_open, "frames": 60},
    {"name": "close", "pos": cube_position, "grip": (grip_open, grip_closed), "frames": 60},
    {"name": "lift", "pos": cube_position + lift_offset, "grip": grip_closed, "frames": 60},
    {"name": "move", "pos": drop_pos + approach_offset, "grip": grip_closed, "frames": 60},
    {"name": "place", "pos": drop_pos, "grip": grip_closed, "frames": 60},
    {"name": "hold", "pos": drop_pos, "grip": grip_closed, "frames": 30},
    {"name": "open", "pos": drop_pos, "grip": (grip_closed, grip_open), "frames": 45},
    {"name": "retract", "pos": drop_pos + approach_offset, "grip": grip_open, "frames": 60},
]

def run_sim_substeps():
    global state_0, state_1
    for _ in range(SIM_SUBSTEPS):
        state_0.clear_forces()
        model.collide(state_0, contacts)
        solver.step(state_in=state_0, state_out=state_1, control=control, contacts=contacts, dt=TIME_STEP)
        state_0, state_1 = state_1, state_0

viewer = create_viewer_viser("franka", model, quiet=True, overwrite=False)
viewer.set_model(model)
viewer.set_camera(wp.vec3(0.5, 0.0, 0.5), -15, -140)


IK_ITERS = 24
ARM_DOFS = _config.ARM_DOF_COUNT
DOFS = _config.DOF_COUNT

sim_time = 0.0

graph = None
if wp.get_device().is_cuda:
    with wp.ScopedCapture() as capture:
        run_sim_substeps()
    graph = capture.graph

phase_bar = tqdm(sequence, desc="phase: init")
for phase in phase_bar:
    phase_bar.set_description(f"phase: {phase['name']}")
    target_pos = phase["pos"]
    frames = int(phase["frames"])
    grip = phase["grip"]

    if isinstance(grip, tuple):
        grip_start, grip_end = float(grip[0]), float(grip[1])
    else:
        grip_start = grip_end = float(grip)

    for frame in range(frames):
        alpha = (frame + 1) / frames if frames > 1 else 1.0
        grip_i = grip_start + (grip_end - grip_start) * alpha

        pos_obj.set_target_positions(wp.array([target_pos], dtype=wp.vec3))
        rot_obj.set_target_rotations(wp.array([fixed_rot[:4]], dtype=wp.vec4))  # type: ignore
        ik_solver.step(joint_q_ik, joint_q_ik, iterations=IK_ITERS)  # type: ignore

        joint_target_pos_view = control.joint_target_pos.reshape((1, -1))  # type: ignore
        wp.copy(dest=joint_target_pos_view[:, :ARM_DOFS], src=joint_q_ik[:, :ARM_DOFS])  # type: ignore
        wp.copy(
            dest=joint_target_pos_view[:, ARM_DOFS:DOFS], src=wp.array([[grip_i, grip_i]], dtype=wp.float32)
        )  # type: ignore

        if graph is not None:
            wp.capture_launch(graph)
        else:
            run_sim_substeps()

        viewer.begin_frame(sim_time)
        viewer.log_state(state_0)
        viewer.end_frame()
        sim_time += FRAME_STEP

viewer.show_notebook()

phase: init:   0%|          | 0/9 [00:00<?, ?it/s]
phase: approach:   0%|          | 0/9 [00:00<?, ?it/s]
phase: approach:  11%|█         | 1/9 [00:03<00:30,  3.75s/it]
phase: descend:  11%|█         | 1/9 [00:03<00:30,  3.75s/it] 
phase: descend:  22%|██▏       | 2/9 [00:07<00:25,  3.66s/it]
phase: close:  22%|██▏       | 2/9 [00:07<00:25,  3.66s/it]  
phase: close:  33%|███▎      | 3/9 [00:11<00:22,  3.73s/it]
phase: lift:  33%|███▎      | 3/9 [00:11<00:22,  3.73s/it] 
phase: lift:  44%|████▍     | 4/9 [00:18<00:25,  5.10s/it]
phase: move:  44%|████▍     | 4/9 [00:18<00:25,  5.10s/it]
phase: move:  56%|█████▌    | 5/9 [00:25<00:23,  5.89s/it]
phase: place:  56%|█████▌    | 5/9 [00:25<00:23,  5.89s/it]
phase: place:  67%|██████▋   | 6/9 [00:34<00:20,  6.98s/it]
phase: hold:  67%|██████▋   | 6/9 [00:34<00:20,  6.98s/it] 
phase: hold:  78%|███████▊  | 7/9 [00:39<00:12,  6.29s/it]
phase: open:  78%|███████▊  | 7/9 [00:39<00:12,  6.29s/it]
phase: open:  89%|████████▉ | 8/9 [00:45<00:06,  6.13s/it]
phase: retract:  89%|████████▉ | 8/9 [00:45<00:06,  6.13s/it]
phase: retract: 100%|██████████| 9/9 [00:48<00:00,  5.17s/it]
phase: retract: 100%|██████████| 9/9 [00:48<00:00,  5.38s/it]