Back to Simulation Environments

NVIDIA Isaac Sim

25 min

NVIDIA Isaac Sim

NVIDIA Isaac Sim is a robotics simulation platform built on NVIDIA Omniverse. It provides photorealistic rendering, GPU-accelerated physics, and tools for generating synthetic data for world model training.

Key Features

1. Photorealistic Rendering

Isaac Sim uses RTX ray tracing for realistic visuals:

  • Global illumination
  • Reflections and refractions
  • Physically-based materials
  • HDR lighting

2. GPU-Accelerated Physics

PhysX 5 provides fast, accurate simulation:

  • Rigid body dynamics
  • Soft body simulation
  • Fluid dynamics
  • Articulated robots

3. Sensor Simulation

Accurate sensor models for:

  • RGB cameras
  • Depth sensors
  • LiDAR
  • IMU
  • Contact sensors

Getting Started

Installation

bash
# Download Isaac Sim from NVIDIA
# Requires NVIDIA RTX GPU

# Launch Isaac Sim
./isaac-sim.sh

# Or use Docker
docker pull nvcr.io/nvidia/isaac-sim:2023.1.0
docker run --gpus all -it nvcr.io/nvidia/isaac-sim:2023.1.0

Basic Scene Setup

python
from omni.isaac.kit import SimulationApp

# Initialize simulation
simulation_app = SimulationApp({"headless": False})

from omni.isaac.core import World
from omni.isaac.core.objects import DynamicCuboid
from omni.isaac.core.robots import Robot

# Create world
world = World(stage_units_in_meters=1.0)

# Add ground plane
world.scene.add_default_ground_plane()

# Add a cube
cube = world.scene.add(
    DynamicCuboid(
        prim_path="/World/Cube",
        name="cube",
        position=[0, 0, 1],
        size=0.1,
        color=[1, 0, 0]
    )
)

# Add a robot
robot = world.scene.add(
    Robot(
        prim_path="/World/Robot",
        name="franka",
        usd_path="path/to/franka.usd"
    )
)

# Run simulation
world.reset()
for i in range(1000):
    world.step(render=True)

Synthetic Data Generation

Domain Randomization

python
from omni.replicator import core as rep

# Setup randomization
with rep.new_layer():
    # Randomize lighting
    light = rep.create.light(
        light_type="dome",
        rotation=rep.distribution.uniform((0, 0, 0), (360, 360, 360)),
        intensity=rep.distribution.uniform(500, 2000)
    )
    
    # Randomize object positions
    cube = rep.get.prims(path_pattern="/World/Cube")
    with cube:
        rep.modify.pose(
            position=rep.distribution.uniform((-1, -1, 0.5), (1, 1, 2)),
            rotation=rep.distribution.uniform((0, 0, 0), (360, 360, 360))
        )
    
    # Randomize textures
    rep.randomizer.materials(
        materials=rep.get.material(path_pattern="/World/Materials/*"),
        seed=42
    )

# Generate dataset
with rep.trigger.on_frame(num_frames=10000):
    rep.WriterRegistry.get("BasicWriter")(
        output_dir="./synthetic_data",
        rgb=True,
        depth=True,
        semantic_segmentation=True
    )

Camera Configuration

python
from omni.isaac.sensor import Camera

# Create camera
camera = Camera(
    prim_path="/World/Camera",
    position=[2, 2, 2],
    frequency=30,
    resolution=(1920, 1080)
)

# Get RGB image
rgb = camera.get_rgba()

# Get depth
depth = camera.get_depth()

# Get semantic segmentation
segmentation = camera.get_semantic_segmentation()

Robot Simulation

Franka Panda Example

python
from omni.isaac.franka import Franka
from omni.isaac.core.controllers import BaseController

# Add Franka robot
franka = world.scene.add(Franka(prim_path="/World/Franka", name="franka"))

# Create controller
class PickPlaceController(BaseController):
    def __init__(self):
        super().__init__(name="pick_place")
        self.phase = "approach"
    
    def forward(self, observations):
        if self.phase == "approach":
            target = observations["target_position"]
            current = observations["end_effector_position"]
            
            if np.linalg.norm(target - current) < 0.01:
                self.phase = "grasp"
            
            return {"joint_velocities": self.compute_ik(target)}
        
        elif self.phase == "grasp":
            return {"gripper_action": "close"}

# Run with controller
controller = PickPlaceController()
while True:
    observations = franka.get_observations()
    actions = controller.forward(observations)
    franka.apply_action(actions)
    world.step()

Integration with World Models

python
class IsaacSimDataCollector:
    """Collect data from Isaac Sim for world model training"""
    
    def __init__(self, world, camera):
        self.world = world
        self.camera = camera
        self.data = []
    
    def collect_episode(self, policy, num_steps=100):
        episode_data = {
            "observations": [],
            "actions": [],
            "rewards": []
        }
        
        self.world.reset()
        
        for _ in range(num_steps):
            # Get observation
            obs = {
                "rgb": self.camera.get_rgba(),
                "depth": self.camera.get_depth(),
                "robot_state": self.get_robot_state()
            }
            
            # Get action from policy
            action = policy(obs)
            
            # Step simulation
            self.apply_action(action)
            self.world.step()
            
            # Get reward
            reward = self.compute_reward()
            
            episode_data["observations"].append(obs)
            episode_data["actions"].append(action)
            episode_data["rewards"].append(reward)
        
        return episode_data

Summary

NVIDIA Isaac Sim provides a powerful platform for robotics simulation and synthetic data generation. Its photorealistic rendering and accurate physics make it ideal for training world models that transfer to the real world.