diff --git a/.github/workflows/fast_tests.yml b/.github/workflows/fast_tests.yml index ad4938970..a39773b4e 100644 --- a/.github/workflows/fast_tests.yml +++ b/.github/workflows/fast_tests.yml @@ -60,12 +60,19 @@ jobs: runs-on: ubuntu-latest env: MUJOCO_GL: egl + HF_HOME: /mnt/cache/.cache/huggingface + HF_LEROBOT_HOME: /mnt/cache/.cache/huggingface/lerobot steps: - uses: actions/checkout@v4 with: persist-credentials: false lfs: true + # NOTE(Steven): Mount to `/mnt` to avoid the limited storage on `/home`. Consider cleaning default SDKs or using self-hosted runners for more space. + # (As of 2024-06-10, the runner's `/home` has only 6.2 GB free—8% of its 72 GB total.) + - name: Setup /mnt storage + run: sudo chown -R $USER:$USER /mnt + # TODO(Steven): Evaluate the need of these dependencies - name: Install apt dependencies run: | diff --git a/.github/workflows/full_tests.yml b/.github/workflows/full_tests.yml index 0155eec13..0dba5e1db 100644 --- a/.github/workflows/full_tests.yml +++ b/.github/workflows/full_tests.yml @@ -58,12 +58,19 @@ jobs: github.event_name == 'workflow_dispatch' env: MUJOCO_GL: egl + HF_HOME: /mnt/cache/.cache/huggingface + HF_LEROBOT_HOME: /mnt/cache/.cache/huggingface/lerobot steps: - uses: actions/checkout@v4 with: lfs: true persist-credentials: false + # NOTE(Steven): Mount to `/mnt` to avoid the limited storage on `/home`. Consider cleaning default SDKs or using self-hosted runners for more space. + # (As of 2024-06-10, the runner's `/home` has only 6.2 GB free—8% of its 72 GB total.) + - name: Setup /mnt storage + run: sudo chown -R $USER:$USER /mnt + - name: Install apt dependencies run: | sudo apt-get update && sudo apt-get install -y build-essential \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f0a84732..4891707ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,11 +83,11 @@ jobs: fi - name: Remove Tags with Git dependencies - # TODO(Steven): Temporary patch to remove libero and pi from PyPi 0.4.0 release due to its reliance on git dependencies. + # TODO(Steven): Temporary patch to remove pi from PyPi 0.4.0 release due to its reliance on git dependencies. run: | echo "::info:: Checking for Git dependencies to remove from pyproject.toml..." - grep -E '@ git\+https|lerobot\[pi\]|lerobot\[libero\]' pyproject.toml | sed 's/^/::warning:: Removing line: /' || true - sed -E -i '/@ git\+https|lerobot\[pi\]|lerobot\[libero\]/d' pyproject.toml + grep -E '@ git\+https|lerobot\[pi\]' pyproject.toml | sed 's/^/::warning:: Removing line: /' || true + sed -E -i '/@ git\+https|lerobot\[pi\]/d' pyproject.toml echo "::info:: Git dependencies removed. Proceeding with build." - name: Install build dependencies diff --git a/.github/workflows/unbound_deps_tests.yml b/.github/workflows/unbound_deps_tests.yml index 902074a83..92271ba8e 100644 --- a/.github/workflows/unbound_deps_tests.yml +++ b/.github/workflows/unbound_deps_tests.yml @@ -45,12 +45,19 @@ jobs: runs-on: ubuntu-latest env: MUJOCO_GL: egl + HF_HOME: /mnt/cache/.cache/huggingface + HF_LEROBOT_HOME: /mnt/cache/.cache/huggingface/lerobot steps: - uses: actions/checkout@v4 with: lfs: true persist-credentials: false + # NOTE(Steven): Mount to `/mnt` to avoid the limited storage on `/home`. Consider cleaning default SDKs or using self-hosted runners for more space. + # (As of 2024-06-10, the runner's `/home` has only 6.2 GB free—8% of its 72 GB total.) + - name: Setup /mnt storage + run: sudo chown -R $USER:$USER /mnt + - name: Install apt dependencies run: | sudo apt-get update && sudo apt-get install -y build-essential \ @@ -70,7 +77,7 @@ jobs: echo "Dependencies unbound:" && cat pyproject.toml - name: Install lerobot with all extras - run: uv sync --all-extras + run: uv sync --all-extras --no-extra groot # TODO(Steven): Make flash-attn optional - name: Run pytest (all extras) run: uv run pytest tests -vv diff --git a/README.md b/README.md index 58a8ccc1b..964af4c1d 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ For a full list of optional dependencies, see: https://pypi.org/project/lerobot/ > [!NOTE] -> For lerobot 0.4.0, if you want to install libero or pi tags, you will have to do: `pip install "lerobot[pi,libero]@git+https://github.com/huggingface/lerobot.git"`. +> For lerobot 0.4.0, if you want to install pi tags, you will have to do: `pip install "lerobot[pi]@git+https://github.com/huggingface/lerobot.git"`. > > This will be solved in the next patch release diff --git a/benchmarks/video/benchmark.py b/benchmarks/video/benchmark.py deleted file mode 100644 index d9e5e62bb..000000000 --- a/benchmarks/video/benchmark.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2024 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import threading -import time -from contextlib import ContextDecorator - - -class TimeBenchmark(ContextDecorator): - """ - Measures execution time using a context manager or decorator. - - This class supports both context manager and decorator usage, and is thread-safe for multithreaded - environments. - - Args: - print: If True, prints the elapsed time upon exiting the context or completing the function. Defaults - to False. - - Examples: - - Using as a context manager: - - >>> benchmark = TimeBenchmark() - >>> with benchmark: - ... time.sleep(1) - >>> print(f"Block took {benchmark.result:.4f} seconds") - Block took approximately 1.0000 seconds - - Using with multithreading: - - ```python - import threading - - benchmark = TimeBenchmark() - - - def context_manager_example(): - with benchmark: - time.sleep(0.01) - print(f"Block took {benchmark.result_ms:.2f} milliseconds") - - - threads = [] - for _ in range(3): - t1 = threading.Thread(target=context_manager_example) - threads.append(t1) - - for t in threads: - t.start() - - for t in threads: - t.join() - ``` - Expected output: - Block took approximately 10.00 milliseconds - Block took approximately 10.00 milliseconds - Block took approximately 10.00 milliseconds - """ - - def __init__(self, print=False): - self.local = threading.local() - self.print_time = print - - def __enter__(self): - self.local.start_time = time.perf_counter() - return self - - def __exit__(self, *exc): - self.local.end_time = time.perf_counter() - self.local.elapsed_time = self.local.end_time - self.local.start_time - if self.print_time: - print(f"Elapsed time: {self.local.elapsed_time:.4f} seconds") - return False - - @property - def result(self): - return getattr(self.local, "elapsed_time", None) - - @property - def result_ms(self): - return self.result * 1e3 diff --git a/benchmarks/video/capture_camera_feed.py b/benchmarks/video/capture_camera_feed.py deleted file mode 100755 index 8f8530532..000000000 --- a/benchmarks/video/capture_camera_feed.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2024 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Capture video feed from a camera as raw images.""" - -import argparse -import datetime as dt -import os -import time -from pathlib import Path - -import cv2 -import rerun as rr - -# see https://rerun.io/docs/howto/visualization/limit-ram -RERUN_MEMORY_LIMIT = os.getenv("LEROBOT_RERUN_MEMORY_LIMIT", "5%") - - -def display_and_save_video_stream(output_dir: Path, fps: int, width: int, height: int, duration: int): - rr.init("lerobot_capture_camera_feed") - rr.spawn(memory_limit=RERUN_MEMORY_LIMIT) - - now = dt.datetime.now() - capture_dir = output_dir / f"{now:%Y-%m-%d}" / f"{now:%H-%M-%S}" - if not capture_dir.exists(): - capture_dir.mkdir(parents=True, exist_ok=True) - - # Opens the default webcam - cap = cv2.VideoCapture(0) - if not cap.isOpened(): - print("Error: Could not open video stream.") - return - - cap.set(cv2.CAP_PROP_FPS, fps) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, width) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height) - - frame_index = 0 - start_time = time.time() - while time.time() - start_time < duration: - ret, frame = cap.read() - - if not ret: - print("Error: Could not read frame.") - break - rr.log("video/stream", rr.Image(frame), static=True) - cv2.imwrite(str(capture_dir / f"frame_{frame_index:06d}.png"), frame) - frame_index += 1 - - # Release the capture - cap.release() - - # TODO(Steven): Add a graceful shutdown via a close() method for the Viewer context, though not currently supported in the Rerun API. - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument( - "--output-dir", - type=Path, - default=Path("outputs/cam_capture/"), - help="Directory where the capture images are written. A subfolder named with the current date & time will be created inside it for each capture.", - ) - parser.add_argument( - "--fps", - type=int, - default=30, - help="Frames Per Second of the capture.", - ) - parser.add_argument( - "--width", - type=int, - default=1280, - help="Width of the captured images.", - ) - parser.add_argument( - "--height", - type=int, - default=720, - help="Height of the captured images.", - ) - parser.add_argument( - "--duration", - type=int, - default=20, - help="Duration in seconds for which the video stream should be captured.", - ) - args = parser.parse_args() - display_and_save_video_stream(**vars(args)) diff --git a/benchmarks/video/run_video_benchmark.py b/benchmarks/video/run_video_benchmark.py index 9f34b2273..064a84b48 100644 --- a/benchmarks/video/run_video_benchmark.py +++ b/benchmarks/video/run_video_benchmark.py @@ -21,11 +21,13 @@ See the provided README.md or run `python benchmark/video/run_video_benchmark.py import argparse import datetime as dt +import itertools import random import shutil from collections import OrderedDict from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path +from threading import Lock import einops import numpy as np @@ -35,13 +37,13 @@ import torch from skimage.metrics import mean_squared_error, peak_signal_noise_ratio, structural_similarity from tqdm import tqdm -from benchmarks.video.benchmark import TimeBenchmark from lerobot.datasets.lerobot_dataset import LeRobotDataset from lerobot.datasets.video_utils import ( - decode_video_frames_torchvision, + decode_video_frames, encode_video_frames, ) from lerobot.utils.constants import OBS_IMAGE +from lerobot.utils.utils import TimerManager BASE_ENCODING = OrderedDict( [ @@ -86,7 +88,7 @@ def load_original_frames(imgs_dir: Path, timestamps: list[float], fps: int) -> t frames = [] for ts in timestamps: idx = int(ts * fps) - frame = PIL.Image.open(imgs_dir / f"frame_{idx:06d}.png") + frame = PIL.Image.open(imgs_dir / f"frame-{idx:06d}.png") frame = torch.from_numpy(np.array(frame)) frame = frame.type(torch.float32) / 255 frame = einops.rearrange(frame, "h w c -> c h w") @@ -97,21 +99,21 @@ def load_original_frames(imgs_dir: Path, timestamps: list[float], fps: int) -> t def save_decoded_frames( imgs_dir: Path, save_dir: Path, frames: torch.Tensor, timestamps: list[float], fps: int ) -> None: - if save_dir.exists() and len(list(save_dir.glob("frame_*.png"))) == len(timestamps): + if save_dir.exists() and len(list(save_dir.glob("frame-*.png"))) == len(timestamps): return save_dir.mkdir(parents=True, exist_ok=True) for i, ts in enumerate(timestamps): idx = int(ts * fps) frame_hwc = (frames[i].permute((1, 2, 0)) * 255).type(torch.uint8).cpu().numpy() - PIL.Image.fromarray(frame_hwc).save(save_dir / f"frame_{idx:06d}_decoded.png") - shutil.copyfile(imgs_dir / f"frame_{idx:06d}.png", save_dir / f"frame_{idx:06d}_original.png") + PIL.Image.fromarray(frame_hwc).save(save_dir / f"frame-{idx:06d}_decoded.png") + shutil.copyfile(imgs_dir / f"frame-{idx:06d}.png", save_dir / f"frame-{idx:06d}_original.png") def save_first_episode(imgs_dir: Path, dataset: LeRobotDataset) -> None: episode_index = 0 ep_num_images = dataset.meta.episodes["length"][episode_index] - if imgs_dir.exists() and len(list(imgs_dir.glob("frame_*.png"))) == ep_num_images: + if imgs_dir.exists() and len(list(imgs_dir.glob("frame-*.png"))) == ep_num_images: return imgs_dir.mkdir(parents=True, exist_ok=True) @@ -125,7 +127,7 @@ def save_first_episode(imgs_dir: Path, dataset: LeRobotDataset) -> None: tqdm(imgs_dataset, desc=f"saving {dataset.repo_id} first episode images", leave=False) ): img = item[img_keys[0]] - img.save(str(imgs_dir / f"frame_{i:06d}.png"), quality=100) + img.save(str(imgs_dir / f"frame-{i:06d}.png"), quality=100) if i >= ep_num_images - 1: break @@ -149,18 +151,6 @@ def sample_timestamps(timestamps_mode: str, ep_num_images: int, fps: int) -> lis return [idx / fps for idx in frame_indexes] -def decode_video_frames( - video_path: str, - timestamps: list[float], - tolerance_s: float, - backend: str, -) -> torch.Tensor: - if backend in ["pyav", "video_reader"]: - return decode_video_frames_torchvision(video_path, timestamps, tolerance_s, backend) - else: - raise NotImplementedError(backend) - - def benchmark_decoding( imgs_dir: Path, video_path: Path, @@ -172,8 +162,8 @@ def benchmark_decoding( num_workers: int = 4, save_frames: bool = False, ) -> dict: - def process_sample(sample: int): - time_benchmark = TimeBenchmark() + def process_sample(sample: int, lock: Lock): + time_benchmark = TimerManager(log=False) timestamps = sample_timestamps(timestamps_mode, ep_num_images, fps) num_frames = len(timestamps) result = { @@ -182,13 +172,13 @@ def benchmark_decoding( "mse_values": [], } - with time_benchmark: + with time_benchmark, lock: frames = decode_video_frames(video_path, timestamps=timestamps, tolerance_s=5e-1, backend=backend) - result["load_time_video_ms"] = time_benchmark.result_ms / num_frames + result["load_time_video_ms"] = (time_benchmark.last * 1000) / num_frames with time_benchmark: original_frames = load_original_frames(imgs_dir, timestamps, fps) - result["load_time_images_ms"] = time_benchmark.result_ms / num_frames + result["load_time_images_ms"] = (time_benchmark.last * 1000) / num_frames frames_np, original_frames_np = frames.numpy(), original_frames.numpy() for i in range(num_frames): @@ -215,8 +205,10 @@ def benchmark_decoding( # A sample is a single set of decoded frames specified by timestamps_mode (e.g. a single frame, 2 frames, etc.). # For each sample, we record metrics (loading time and quality metrics) which are then averaged over all samples. # As these samples are independent, we run them in parallel threads to speed up the benchmark. + # Use a single shared lock for all worker threads + shared_lock = Lock() with ThreadPoolExecutor(max_workers=num_workers) as executor: - futures = [executor.submit(process_sample, i) for i in range(num_samples)] + futures = [executor.submit(process_sample, i, shared_lock) for i in range(num_samples)] for future in tqdm(as_completed(futures), total=num_samples, desc="samples", leave=False): result = future.result() load_times_video_ms.append(result["load_time_video_ms"]) @@ -358,24 +350,27 @@ def main( imgs_dir = output_dir / "images" / dataset.repo_id.replace("/", "_") # We only use the first episode save_first_episode(imgs_dir, dataset) - for key, values in tqdm(encoding_benchmarks.items(), desc="encodings (g, crf)", leave=False): - for value in tqdm(values, desc=f"encodings ({key})", leave=False): - encoding_cfg = BASE_ENCODING.copy() - encoding_cfg["vcodec"] = video_codec - encoding_cfg["pix_fmt"] = pixel_format + for duet in [ + dict(zip(encoding_benchmarks.keys(), unique_combination, strict=False)) + for unique_combination in itertools.product(*encoding_benchmarks.values()) + ]: + encoding_cfg = BASE_ENCODING.copy() + encoding_cfg["vcodec"] = video_codec + encoding_cfg["pix_fmt"] = pixel_format + for key, value in duet.items(): encoding_cfg[key] = value - args_path = Path("_".join(str(value) for value in encoding_cfg.values())) - video_path = output_dir / "videos" / args_path / f"{repo_id.replace('/', '_')}.mp4" - benchmark_table += benchmark_encoding_decoding( - dataset, - video_path, - imgs_dir, - encoding_cfg, - decoding_benchmarks, - num_samples, - num_workers, - save_frames, - ) + args_path = Path("_".join(str(value) for value in encoding_cfg.values())) + video_path = output_dir / "videos" / args_path / f"{repo_id.replace('/', '_')}.mp4" + benchmark_table += benchmark_encoding_decoding( + dataset, + video_path, + imgs_dir, + encoding_cfg, + decoding_benchmarks, + num_samples, + num_workers, + save_frames, + ) # Save intermediate results benchmark_df = pd.DataFrame(benchmark_table, columns=headers) @@ -409,9 +404,9 @@ if __name__ == "__main__": nargs="*", default=[ "lerobot/pusht_image", - "aliberts/aloha_mobile_shrimp_image", - "aliberts/paris_street", - "aliberts/kitchen", + "lerobot/aloha_mobile_shrimp_image", + "lerobot/paris_street", + "lerobot/kitchen", ], help="Datasets repo-ids to test against. First episodes only are used. Must be images.", ) @@ -419,7 +414,7 @@ if __name__ == "__main__": "--vcodec", type=str, nargs="*", - default=["libx264", "hevc", "libsvtav1"], + default=["h264", "hevc", "libsvtav1"], help="Video codecs to be tested", ) parser.add_argument( @@ -468,7 +463,7 @@ if __name__ == "__main__": "--backends", type=str, nargs="*", - default=["pyav", "video_reader"], + default=["torchcodec", "pyav"], help="Torchvision decoding backend to be tested.", ) parser.add_argument( diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 3e36a0d98..f9f792113 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -9,14 +9,14 @@ title: Imitation Learning for Robots - local: cameras title: Cameras + - local: bring_your_own_policies + title: Bring Your Own Policies - local: integrate_hardware title: Bring Your Own Hardware - local: hilserl title: Train a Robot with RL - local: hilserl_sim title: Train RL in Simulation - - local: async - title: Use Async Inference - local: multi_gpu_training title: Multi GPU training title: "Tutorials" @@ -39,10 +39,20 @@ title: π₀.₅ (Pi05) - local: groot title: NVIDIA GR00T N1.5 + - local: xvla + title: X-VLA title: "Policies" - sections: - - local: il_sim - title: Imitation Learning in Sim + - local: async + title: Use Async Inference + - local: rtc + title: Real-Time Chunking (RTC) + title: "Inference" +- sections: + - local: envhub + title: Environments from the Hub + - local: envhub_leisaac + title: Control & Train Robots in Sim (LeIsaac) - local: libero title: Using Libero - local: metaworld @@ -57,6 +67,8 @@ title: Implement your own processor - local: processors_robots_teleop title: Processors for Robots and Teleoperators + - local: env_processor + title: Environment Processors title: "Robot Processors" - sections: - local: so101 @@ -71,6 +83,10 @@ title: Hope Jr - local: reachy2 title: Reachy 2 + - local: unitree_g1 + title: Unitree G1 + - local: earthrover_mini_plus + title: Earth Rover Mini title: "Robots" - sections: - local: phone_teleop diff --git a/docs/source/async.mdx b/docs/source/async.mdx index be10f8baf..9dd87472c 100644 --- a/docs/source/async.mdx +++ b/docs/source/async.mdx @@ -196,7 +196,7 @@ client_cfg = RobotClientConfig( server_address="localhost:8080", policy_device="mps", policy_type="smolvla", - pretrained_name_or_path="fracapuano/smolvla_async", + pretrained_name_or_path="/smolvla_async", chunk_size_threshold=0.5, actions_per_chunk=50, # make sure this is less than the max actions of the policy ) @@ -278,7 +278,7 @@ We found the default values of `actions_per_chunk` and `chunk_size_threshold` to 2. **Adjust your `fps` based on inference latency.** While the server generates a new action chunk, the client is not idle and is stepping through its current action queue. If the two processes happen at fundamentally different speeds, the client might end up with an empty queue. As such, you should reduce your fps if you consistently run out of actions in queue. 3. **Adjust `chunk_size_threshold`**. - Values closer to `0.0` result in almost sequential behavior. Values closer to `1.0` → send observation every step (more bandwidth, relies on good world-model). - - We found values around 0.5-0.6 to work well. If you want to tweak this, spin up a `RobotClient` setting the `--debug-visualize-queue-size` to `True`. This will plot the action queue size evolution at runtime, and you can use it to find the value of `chunk_size_threshold` that works best for your setup. + - We found values around 0.5-0.6 to work well. If you want to tweak this, spin up a `RobotClient` setting the `--debug_visualize_queue_size` to `True`. This will plot the action queue size evolution at runtime, and you can use it to find the value of `chunk_size_threshold` that works best for your setup.

The action queue size is plotted at runtime when the - `--debug-visualize-queue-size` flag is passed, for various levels of + `--debug_visualize_queue_size` flag is passed, for various levels of `chunk_size_threshold` (`g` in the SmolVLA paper).

diff --git a/docs/source/bring_your_own_policies.mdx b/docs/source/bring_your_own_policies.mdx new file mode 100644 index 000000000..0ff098708 --- /dev/null +++ b/docs/source/bring_your_own_policies.mdx @@ -0,0 +1,175 @@ +# Bring Your Own Policies + +This tutorial explains how to integrate your own custom policy implementations into the LeRobot ecosystem, allowing you to leverage all LeRobot tools for training, evaluation, and deployment while using your own algorithms. + +## Step 1: Create a Policy Package + +Your custom policy should be organized as an installable Python package following LeRobot's plugin conventions. + +### Package Structure + +Create a package with the prefix `lerobot_policy_` (IMPORTANT!) followed by your policy name: + +```bash +lerobot_policy_my_custom_policy/ +├── pyproject.toml +└── src/ + └── lerobot_policy_my_custom_policy/ + ├── __init__.py + ├── configuration_my_custom_policy.py + ├── modeling_my_custom_policy.py + └── processor_my_custom_policy.py +``` + +### Package Configuration + +Set up your `pyproject.toml`: + +```toml +[project] +name = "lerobot_policy_my_custom_policy" +version = "0.1.0" +dependencies = [ + # your policy-specific dependencies +] +requires-python = ">= 3.11" + +[build-system] +build-backend = # your-build-backend +requires = # your-build-system +``` + +## Step 2: Define the Policy Configuration + +Create a configuration class that inherits from `PreTrainedConfig` and registers your policy type: + +```python +# configuration_my_custom_policy.py +from dataclasses import dataclass, field +from lerobot.configs.policies import PreTrainedConfig +from lerobot.configs.types import NormalizationMode + +@PreTrainedConfig.register_subclass("my_custom_policy") +@dataclass +class MyCustomPolicyConfig(PreTrainedConfig): + """Configuration class for MyCustomPolicy. + + Args: + n_obs_steps: Number of observation steps to use as input + horizon: Action prediction horizon + n_action_steps: Number of action steps to execute + hidden_dim: Hidden dimension for the policy network + # Add your policy-specific parameters here + """ + # ...PreTrainedConfig fields... + pass + + def __post_init__(self): + super().__post_init__() + # Add any validation logic here + + def validate_features(self) -> None: + """Validate input/output feature compatibility.""" + # Implement validation logic for your policy's requirements + pass +``` + +## Step 3: Implement the Policy Class + +Create your policy implementation by inheriting from LeRobot's base `PreTrainedPolicy` class: + +```python +# modeling_my_custom_policy.py +import torch +import torch.nn as nn +from typing import Dict, Any + +from lerobot.policies.pretrained import PreTrainedPolicy +from .configuration_my_custom_policy import MyCustomPolicyConfig + +class MyCustomPolicy(PreTrainedPolicy): + config_class = MyCustomPolicyConfig + name = "my_custom_policy" + + def __init__(self, config: MyCustomPolicyConfig, dataset_stats: Dict[str, Any] = None): + super().__init__(config, dataset_stats) + ... +``` + +## Step 4: Add Data Processors + +Create processor functions: + +```python +# processor_my_custom_policy.py +from typing import Dict, Any +import torch + + +def make_my_custom_policy_pre_post_processors( + config, +) -> tuple[ + PolicyProcessorPipeline[dict[str, Any], dict[str, Any]], + PolicyProcessorPipeline[PolicyAction, PolicyAction], +]: + """Create preprocessing and postprocessing functions for your policy.""" + pass # Define your preprocessing and postprocessing logic here + +``` + +## Step 5: Package Initialization + +Expose your classes in the package's `__init__.py`: + +```python +# __init__.py +"""Custom policy package for LeRobot.""" + +try: + import lerobot # noqa: F401 +except ImportError: + raise ImportError( + "lerobot is not installed. Please install lerobot to use this policy package." + ) + +from .configuration_my_custom_policy import MyCustomPolicyConfig +from .modeling_my_custom_policy import MyCustomPolicy +from .processor_my_custom_policy import make_my_custom_policy_pre_post_processors + +__all__ = [ + "MyCustomPolicyConfig", + "MyCustomPolicy", + "make_my_custom_policy_pre_post_processors", +] +``` + +## Step 6: Installation and Usage + +### Install Your Policy Package + +```bash +cd lerobot_policy_my_custom_policy +pip install -e . + +# Or install from PyPI if published +pip install lerobot_policy_my_custom_policy +``` + +### Use Your Policy + +Once installed, your policy automatically integrates with LeRobot's training and evaluation tools: + +```bash +lerobot-train \ + --policy.type my_custom_policy \ + --env.type pusht \ + --steps 200000 +``` + +## Examples and Community Contributions + +Check out these example policy implementations: + +- [DiTFlow Policy](https://github.com/danielsanjosepro/lerobot_policy_ditflow) - Diffusion Transformer policy with flow-matching objective. Try it out in this example: [DiTFlow Example](https://github.com/danielsanjosepro/test_lerobot_policy_ditflow) + +Share your policy implementations with the community! 🤗 diff --git a/docs/source/earthrover_mini_plus.mdx b/docs/source/earthrover_mini_plus.mdx new file mode 100644 index 000000000..7e27eb93e --- /dev/null +++ b/docs/source/earthrover_mini_plus.mdx @@ -0,0 +1,206 @@ +# EarthRover Mini Plus + +The EarthRover Mini Plus is a fully open source mobile robot that connects through the cloud using the Frodobots SDK. This lets you control the robot and record datasets for training AI models. + +## What You Need + +### Hardware + +- EarthRover Mini robot +- Computer with Python 3.10 or newer +- Internet connection + +### Setting Up the Frodobots SDK + +The robot needs the [Frodobots SDK](https://github.com/Frodobots/earth-rovers-sdk) running on your computer. Here's how: + +1. Download and install the SDK: + +```bash +git clone https://github.com/Frodobots/earth-rovers-sdk.git +cd earth-rovers-sdk +pip install -r requirements.txt +``` + +2. Start the SDK: + +```bash +hypercorn main:app --reload +``` + +3. Open your web browser and go to `http://localhost:8000`, then click "Join" + +The SDK gives you: + +- Live video from front and rear cameras + +> [!IMPORTANT] +> The SDK must be running before you can use the robot. + +## Install LeRobot + +Follow our [Installation Guide](./installation) to install LeRobot. + +In addition to the base installation, install the EarthRover Mini dependencies: + +```bash +pip install -e . +``` + +## How It Works + +The robot uses the internet to communicate: + +- **Movement commands**: Sent through the SDK +- **Camera video**: Received from the SDK +- **Robot info**: Battery, location, speed from the SDK + +You don't need to plug anything in - it all works through the SDK. + +## Calibration + +No calibration needed! The robot is ready to use as soon as the SDK is running. + +## Controlling the Robot + +You control the robot using your keyboard - just like playing a video game with WASD keys. + +### Keyboard Controls + +| Key | Action | +| --- | -------------------------------- | +| W | Move forward | +| S | Move backward | +| A | Turn left (with forward motion) | +| D | Turn right (with forward motion) | +| Q | Rotate left in place | +| E | Rotate right in place | +| X | Stop all movement | +| +/= | Increase speed | +| - | Decrease speed | +| ESC | Disconnect | + +### Speed Settings + +You can adjust how fast the robot moves: + +- **Forward/backward speed**: Default is full speed (1.0) +- **Turning speed**: Default is full speed (1.0) +- **Speed changes**: Use +/- keys to adjust by 0.1 each time + +### Try It Out + +Test driving the robot before recording data: + +```python +from lerobot.robots.earthrover_mini_plus import EarthRoverMiniPlus, EarthRoverMiniPlusConfig +from lerobot.teleoperators.keyboard import KeyboardRoverTeleop, KeyboardRoverTeleopConfig + +# Initialize robot +robot_config = EarthRoverMiniPlusConfig() +robot = EarthRoverMiniPlus(robot_config) + +# Initialize teleoperator +teleop_config = KeyboardRoverTeleopConfig( + linear_speed=1.0, + angular_speed=1.0, + speed_increment=0.1 +) +teleop = KeyboardRoverTeleop(teleop_config) + +# Connect +robot.connect() +teleop.connect() + +# Teleoperate (use keyboard controls) +try: + while True: + action = teleop.get_action() + robot.send_action(action) +except KeyboardInterrupt: + pass +finally: + robot.disconnect() + teleop.disconnect() +``` + +> [!TIP] +> If you're using a Mac, you might need to give Terminal permission to access your keyboard for teleoperation. Go to System Preferences > Security & Privacy > Input Monitoring and check the box for Terminal. + +## Recording Data + +Once you can drive the robot well, you can start recording data to train AI models. The system records: + +- **What you do**: How you move the robot (forward, backward, turning) +- **What the robot sees**: + - Videos from both cameras + - Robot speed and direction + - Battery level and location + - GPS position and signal + - Other sensor data +- **When it happened**: Timestamps for everything + +### Setting Up Hugging Face + +We use Hugging Face to store your data online. First, log in with your token from [Hugging Face settings](https://huggingface.co/settings/tokens): + +```bash +huggingface-cli login --token ${HUGGINGFACE_TOKEN} --add-to-git-credential +``` + +Store your Hugging Face username: + +```bash +HF_USER=$(huggingface-cli whoami | head -n 1) +echo $HF_USER +``` + +### Start Recording + +Use the standard recording command: + +```bash +python src/lerobot/scripts/lerobot_record.py \ + --robot.type=earthrover_mini_plus \ + --teleop.type=keyboard_rover \ + --dataset.repo_id=your_username/dataset_name \ + --dataset.num_episodes=2 \ + --dataset.fps=10 \ + --dataset.single_task="Navigate around obstacles" \ + --display_data=true +``` + +Replace `your_username/dataset_name` with your Hugging Face username and a name for your dataset. + +### What Gets Saved + +Your dataset includes: + +**Your Actions (2 things)**: + +- How much you moved forward/backward +- How much you turned left/right + +**Robot Observations (12 things)**: + +- Front camera video +- Rear camera video +- Current speed +- Battery level +- Which way the robot is facing +- GPS location (latitude, longitude, signal strength) +- Network signal strength +- Vibration level +- Lamp status (on/off) + +### Where Your Data Goes + +On your computer: `~/.cache/huggingface/lerobot/{repo-id}` + +After recording, your data automatically uploads to your Hugging Face page: + +```bash +echo https://huggingface.co/datasets/${HF_USER}/earthrover-navigation +``` + +Your dataset will be tagged with `LeRobot` for community discovery. diff --git a/docs/source/env_processor.mdx b/docs/source/env_processor.mdx new file mode 100644 index 000000000..8dbf315c7 --- /dev/null +++ b/docs/source/env_processor.mdx @@ -0,0 +1,418 @@ +# Environment Processors + +Environment processors are a critical layer in LeRobot's data processing architecture that handle **environment-specific** transformations, separate from policy-specific processing. This separation of concerns enables cleaner code, better modularity, and easier experimentation with different environments and policies. + +## Why Environment Processors? + +When working with different robot environments (LIBERO, MetaWorld, Aloha, etc.), each environment often has unique data formats, coordinate systems, and conventions that need standardization **before** policy processing. Without environment processors, these transformations would be: + +1. **Hardcoded in environment code** - Making it difficult to experiment with different state representations +2. **Duplicated across policies** - Each policy would need to handle environment-specific quirks +3. **Mixed with policy logic** - Violating separation of concerns and making debugging harder + +Environment processors solve this by providing a **dedicated processing layer** between raw environment observations and policy inputs. + +## The Processing Pipeline + +Here's how data flows through the complete processing pipeline during evaluation: + +```python +# In lerobot_eval.py rollout() function: + +# 1. Raw environment observation (numpy arrays, various formats) +raw_observation = env.step(action) + +# 2. Convert numpy to torch, normalize images [0,1] +observation = preprocess_observation(raw_observation) + +# 3. Add task metadata (for multi-task environments) +observation = add_envs_task(env, observation) + +# 4. ENVIRONMENT-SPECIFIC preprocessing (NEW!) +# - Flatten robot states +# - Rotate images to match dataset conventions +# - Handle environment-specific coordinate systems +observation = env_preprocessor(observation) + +# 5. POLICY-SPECIFIC preprocessing +# - Normalize with dataset statistics +# - Add batch dimensions +# - Move to GPU +# - Tokenize language instructions +observation = preprocessor(observation) + +# 6. Policy inference +action = policy.select_action(observation) + +# 7. POLICY-SPECIFIC postprocessing +# - Unnormalize actions +# - Remove batch dimensions +action = postprocessor(action) + +# 8. ENVIRONMENT-SPECIFIC postprocessing (NEW!) +# - Convert action formats if needed +# - Apply environment-specific constraints +action_transition = {"action": action} +action_transition = env_postprocessor(action_transition) +action = action_transition["action"] + +# 9. Execute in environment +env.step(action) +``` + +## The Benefits + +### 1. **Separation of Concerns** + +Environment processors handle transformations specific to the **environment's data format**, while policy processors handle transformations specific to the **model's requirements**. + +```python +# ❌ Before: Mixed concerns +class LiberoVLAPolicy: + def preprocess(self, obs): + # Environment-specific: Flatten robot state (shouldn't be in policy!) + state = self._flatten_robot_state(obs["robot_state"]) + # Policy-specific: Normalize with dataset stats + state = self.normalizer(state) + return state + +# ✅ After: Clear separation +# Environment processor: Handles LIBERO's nested robot state +env_preprocessor = LiberoProcessorStep() # Flattens robot_state + +# Policy processor: Handles model requirements +policy_preprocessor = NormalizerProcessorStep(stats=dataset_stats) +``` + +### 2. **Flexibility and Reusability** + +The same policy can work with different environment processors, and the same environment processor can work with different policies: + +```python +# Use SmolVLA policy with LIBERO environment +libero_preprocessor, libero_postprocessor = make_env_pre_post_processors(libero_cfg) +smolvla_preprocessor, smolvla_postprocessor = make_pre_post_processors(smolvla_cfg) + +# Or use ACT policy with the same LIBERO environment +libero_preprocessor, libero_postprocessor = make_env_pre_post_processors(libero_cfg) +act_preprocessor, act_postprocessor = make_pre_post_processors(act_cfg) +``` + +### 3. **Easier Experimentation** + +Want to try different state representations for LIBERO? Just create a new processor: + +```python +# Original: 8D state (pos + quat→axisangle + gripper) +@ProcessorStepRegistry.register("libero_processor") +class LiberoProcessorStep(ObservationProcessorStep): + def _process_observation(self, obs): + eef_pos = robot_state["eef"]["pos"] # 3D + eef_axisangle = quat2axisangle(quat) # 3D + gripper = robot_state["gripper"]["qpos"] # 2D + state = torch.cat([eef_pos, eef_axisangle, gripper], dim=-1) # 8D + return state + +# Experiment: Add velocity for better control +@ProcessorStepRegistry.register("libero_velocity_processor") +class LiberoVelocityProcessorStep(ObservationProcessorStep): + def _process_observation(self, obs): + # Include velocities for 14D state + eef_pos = robot_state["eef"]["pos"] # 3D + eef_axisangle = quat2axisangle(quat) # 3D + eef_vel = robot_state["eef"]["vel"] # 3D (NEW) + gripper_pos = robot_state["gripper"]["qpos"] # 2D + gripper_vel = robot_state["gripper"]["qvel"] # 3D (NEW) + state = torch.cat([eef_pos, eef_axisangle, eef_vel, + gripper_pos, gripper_vel], dim=-1) # 14D + return state +``` + +### 4. **Cleaner Environment Code** + +Environments expose **all available data** without needing to know what downstream models will use: + +```python +# LIBERO environment exposes full robot state +observation = { + "pixels": {"image": img, "image2": img2}, + "robot_state": { + "eef": {"pos": ..., "quat": ..., "vel": ..., "mat": ..., "axisangle": ...}, + "gripper": {"qpos": ..., "qvel": ...}, + "joints": {"pos": ..., "vel": ...} + } +} + +# Environment processor decides what to use +# Policy processor handles model-specific transformations +``` + +## Using Environment Processors + +### Factory Function + +The `make_env_pre_post_processors` function follows the same pattern as `make_pre_post_processors` for policies: + +```python +from lerobot.envs.factory import make_env_pre_post_processors +from lerobot.envs.configs import LiberoEnv, PushtEnv + +# For LIBERO: Returns LiberoProcessorStep in preprocessor +libero_cfg = LiberoEnv(task="libero_spatial", camera_name=["agentview"]) +env_preprocessor, env_postprocessor = make_env_pre_post_processors(libero_cfg) + +# For other environments: Returns identity processors (no-op) +pusht_cfg = PushtEnv() +env_preprocessor, env_postprocessor = make_env_pre_post_processors(pusht_cfg) +``` + +### Implementation in `envs/factory.py` + +```python +def make_env_pre_post_processors( + env_cfg: EnvConfig, +) -> tuple[ + PolicyProcessorPipeline[dict[str, Any], dict[str, Any]], + PolicyProcessorPipeline[dict[str, Any], dict[str, Any]], +]: + """ + Create preprocessor and postprocessor pipelines for environment observations. + + Args: + env_cfg: The configuration of the environment. + + Returns: + A tuple containing: + - preprocessor: Pipeline that processes environment observations + - postprocessor: Pipeline that processes environment outputs + """ + # For LIBERO environments, add the LiberoProcessorStep to preprocessor + if isinstance(env_cfg, LiberoEnv) or "libero" in env_cfg.type: + preprocessor = PolicyProcessorPipeline(steps=[LiberoProcessorStep()]) + else: + # For all other environments, return an identity preprocessor + preprocessor = PolicyProcessorPipeline(steps=[]) + + # Postprocessor is currently identity for all environments + # Future: Could add environment-specific action transformations + postprocessor = PolicyProcessorPipeline(steps=[]) + + return preprocessor, postprocessor +``` + +### Integration in Evaluation + +In `lerobot_eval.py`, the environment processors are created once and used throughout: + +```python +def eval_main(cfg: EvalPipelineConfig): + # Create environment + envs = make_env(cfg.env, n_envs=cfg.eval.batch_size) + + # Create policy + policy = make_policy(cfg=cfg.policy, env_cfg=cfg.env) + + # Create policy processors + preprocessor, postprocessor = make_pre_post_processors( + policy_cfg=cfg.policy, + pretrained_path=cfg.policy.pretrained_path, + ) + + # Create environment processors (NEW!) + env_preprocessor, env_postprocessor = make_env_pre_post_processors(env_cfg=cfg.env) + + # Run evaluation with both processor types + eval_policy_all( + envs=envs, + policy=policy, + env_preprocessor=env_preprocessor, # Environment-specific + env_postprocessor=env_postprocessor, # Environment-specific + preprocessor=preprocessor, # Policy-specific + postprocessor=postprocessor, # Policy-specific + n_episodes=cfg.eval.n_episodes, + ) +``` + +## Example: LIBERO Environment Processor + +The `LiberoProcessorStep` demonstrates a real-world environment processor: + +```python +from lerobot.processor.pipeline import ObservationProcessorStep + +@dataclass +@ProcessorStepRegistry.register(name="libero_processor") +class LiberoProcessorStep(ObservationProcessorStep): + """ + Processes LIBERO observations into the LeRobot format. + + **State Processing:** + - Extracts end-effector position (3D) + - Converts quaternion to axis-angle representation (3D) + - Extracts gripper joint positions (2D) + - Concatenates into 8D state vector + + **Image Processing:** + - Rotates images 180° to match HuggingFaceVLA/libero convention + """ + + def _process_observation(self, observation): + processed_obs = observation.copy() + + # Process images: Flip 180° for camera convention + for key in list(processed_obs.keys()): + if key.startswith("observation.images."): + img = processed_obs[key] + img = torch.flip(img, dims=[2, 3]) # Flip H and W + processed_obs[key] = img + + # Process robot_state: Flatten to 8D vector + if "observation.robot_state" in processed_obs: + robot_state = processed_obs.pop("observation.robot_state") + + eef_pos = robot_state["eef"]["pos"] # (B, 3) + eef_quat = robot_state["eef"]["quat"] # (B, 4) + gripper_qpos = robot_state["gripper"]["qpos"] # (B, 2) + + # Convert quaternion to axis-angle + eef_axisangle = self._quat2axisangle(eef_quat) # (B, 3) + + # Concatenate into single state vector + state = torch.cat((eef_pos, eef_axisangle, gripper_qpos), dim=-1) + state = state.float() + + processed_obs["observation.state"] = state + + return processed_obs +``` + +### Why These Transformations? + +1. **Image Rotation**: The HuggingFaceVLA/libero dataset has images rotated 180° from the raw LIBERO simulator. The processor handles this convention mismatch so policies trained on the dataset work seamlessly. + +2. **State Flattening**: The raw LIBERO environment exposes nested dictionaries with all available state information (position, quaternion, velocity, matrix representation, etc.). The processor: + - Selects the relevant components (pos, quat, gripper) + - Converts quaternion to axis-angle (more suitable for learning) + - Flattens to a single 8D vector that policies expect + +3. **Flexibility**: The environment still exposes **all** raw data. If you want to try different state representations (e.g., including velocities, using matrix representation instead of axis-angle), you can create a new processor without modifying the environment code. + +## Adding Environment Processors for New Environments + +To add environment processors for a new environment: + +### 1. Create the Processor Step + +```python +# In src/lerobot/processor/env_processor.py + +@dataclass +@ProcessorStepRegistry.register(name="myenv_processor") +class MyEnvProcessorStep(ObservationProcessorStep): + """Process observations from MyEnv.""" + + def _process_observation(self, observation): + processed = observation.copy() + + # Your environment-specific transformations + if "myenv.specific.state" in processed: + state = processed.pop("myenv.specific.state") + # Transform to standard format + processed["observation.state"] = self._transform_state(state) + + return processed +``` + +### 2. Update the Factory + +```python +# In src/lerobot/envs/factory.py + +def make_env_pre_post_processors(env_cfg: EnvConfig): + if isinstance(env_cfg, LiberoEnv) or "libero" in env_cfg.type: + preprocessor = PolicyProcessorPipeline(steps=[LiberoProcessorStep()]) + elif isinstance(env_cfg, MyEnvConfig) or "myenv" in env_cfg.type: + preprocessor = PolicyProcessorPipeline(steps=[MyEnvProcessorStep()]) + else: + preprocessor = PolicyProcessorPipeline(steps=[]) + + postprocessor = PolicyProcessorPipeline(steps=[]) + return preprocessor, postprocessor +``` + +### 3. Use in Evaluation + +No changes needed! The evaluation script automatically uses the appropriate processor: + +```bash +lerobot-eval \ + --policy.path=lerobot/my_policy \ + --env.type=myenv \ # Automatically uses MyEnvProcessorStep + --eval.n_episodes=10 +``` + +## Future: Environment Postprocessors + +Currently, postprocessors are identity (no-op) for all environments. Future use cases include: + +### Action Space Transformations + +```python +@dataclass +class MyEnvActionPostprocessor(ProcessorStep): + """Convert policy actions to environment-specific format.""" + + def __call__(self, transition: EnvTransition) -> EnvTransition: + action = transition["action"] + + # Example: Convert from Cartesian to joint space + if self.action_space == "joint": + action = self.ik_solver(action) + + # Example: Apply environment-specific safety limits + action = torch.clamp(action, self.min_action, self.max_action) + + transition["action"] = action + return transition +``` + +### Coordinate System Conversions + +```python +@dataclass +class CoordinateTransformPostprocessor(ProcessorStep): + """Transform actions between coordinate systems.""" + + def __call__(self, transition: EnvTransition) -> EnvTransition: + action = transition["action"] + + # Example: Policy outputs in world frame, env expects base frame + action = self.world_to_base_transform(action) + + transition["action"] = action + return transition +``` + +## Best Practices + +1. **Keep environment processors simple**: They should only handle environment-specific data format issues, not complex learning-related transformations. + +2. **Use policy processors for model requirements**: Normalization, batching, device placement, and tokenization belong in policy processors. + +3. **Expose all data from environments**: Let processors decide what to use rather than hardcoding choices in the environment. + +4. **Document conventions**: Clearly document any coordinate system conventions, camera orientations, or data formats that your processor handles. + +5. **Test independently**: Environment processors should be testable without loading full policies or environments. + +## Summary + +Environment processors provide a **clean separation** between environment-specific data transformations and policy-specific model requirements. This architecture: + +- ✅ Enables easy experimentation with different state representations +- ✅ Allows policies to work seamlessly across different environments +- ✅ Keeps environment code focused on simulation/hardware interface +- ✅ Makes processor pipelines more maintainable and debuggable +- ✅ Follows the single responsibility principle + +The key insight: **Environments define data formats, processors standardize them, policies consume standardized data.** Each layer has a clear, focused responsibility. diff --git a/docs/source/envhub.mdx b/docs/source/envhub.mdx new file mode 100644 index 000000000..ba6464460 --- /dev/null +++ b/docs/source/envhub.mdx @@ -0,0 +1,424 @@ +# Loading Environments from the Hub + +The **EnvHub** feature allows you to load simulation environments directly from the Hugging Face Hub with a single line of code. This unlocks a powerful new model for collaboration: instead of environments being locked away inside monolithic libraries, anyone can publish custom environments and share them with the community. + +## Overview + +With EnvHub, you can: + +- Load environments from the Hub instantly +- Share your custom simulation tasks with the community +- Version control your environments using Git +- Distribute complex physics simulations without packaging hassles + +## Quick Start + +Loading an environment from the Hub is as simple as: + +```python +from lerobot.envs.factory import make_env + +# Load a hub environment (requires explicit consent to run remote code) +env = make_env("lerobot/cartpole-env", trust_remote_code=True) +``` + + + **Security Notice**: Loading environments from the Hub executes Python code + from third-party repositories. Only use `trust_remote_code=True` with + repositories you trust. We strongly recommend pinning to a specific commit + hash for reproducibility and security. + + +## What is EnvHub? + +EnvHub is a framework that allows researchers and developers to: + +1. **Publish environments** to the Hugging Face Hub as Git repositories +2. **Load environments** dynamically without installing them as packages +3. **Version and track** environment changes using Git semantics +4. **Discover** new simulation tasks shared by the community + +This design means you can go from discovering an interesting environment on the Hub to running experiments in seconds, without worrying about dependency conflicts or complex installation procedures. + +## Repository Structure + +To make your environment loadable from the Hub, your repository must contain at minimum: + +### Required Files + +**`env.py`** (or custom Python file) + +- Must expose a `make_env(n_envs: int, use_async_envs: bool)` function +- This function should return one of: + - A `gym.vector.VectorEnv` (most common) + - A single `gym.Env` (will be automatically wrapped) + - A dict mapping `{suite_name: {task_id: VectorEnv}}` (for multi-task benchmarks) + +### Optional Files + +**`requirements.txt`** + +- List any additional dependencies your environment needs +- Users will need to install these manually before loading your environment + +**`README.md`** + +- Document your environment: what task it implements, observation/action spaces, rewards, etc. +- Include usage examples and any special setup instructions + +**`.gitignore`** + +- Exclude unnecessary files from your repository + +### Example Repository Structure + +``` +my-environment-repo/ +├── env.py # Main environment definition (required) +├── requirements.txt # Dependencies (optional) +├── README.md # Documentation (recommended) +├── assets/ # Images, videos, etc. (optional) +│ └── demo.gif +└── configs/ # Config files if needed (optional) + └── task_config.yaml +``` + +## Creating Your Environment Repository + +### Step 1: Define Your Environment + +Create an `env.py` file with a `make_env` function: + +```python +# env.py +import gymnasium as gym + +def make_env(n_envs: int = 1, use_async_envs: bool = False): + """ + Create vectorized environments for your custom task. + + Args: + n_envs: Number of parallel environments + use_async_envs: Whether to use AsyncVectorEnv or SyncVectorEnv + + Returns: + gym.vector.VectorEnv or dict mapping suite names to vectorized envs + """ + def _make_single_env(): + # Create your custom environment + return gym.make("CartPole-v1") + + # Choose vector environment type + env_cls = gym.vector.AsyncVectorEnv if use_async_envs else gym.vector.SyncVectorEnv + + # Create vectorized environment + vec_env = env_cls([_make_single_env for _ in range(n_envs)]) + + return vec_env +``` + +### Step 2: Test Locally + +Before uploading, test your environment locally: + +```python +from lerobot.envs.utils import _load_module_from_path, _call_make_env, _normalize_hub_result + +# Load your module +module = _load_module_from_path("./env.py") + +# Test the make_env function +result = _call_make_env(module, n_envs=2, use_async_envs=False) +normalized = _normalize_hub_result(result) + +# Verify it works +suite_name = next(iter(normalized)) +env = normalized[suite_name][0] +obs, info = env.reset() +print(f"Observation shape: {obs.shape if hasattr(obs, 'shape') else type(obs)}") +env.close() +``` + +### Step 3: Upload to the Hub + +Upload your repository to Hugging Face: + +```bash +# Install huggingface_hub if needed +pip install huggingface_hub + +# Login to Hugging Face +huggingface-cli login + +# Create a new repository +huggingface-cli repo create my-custom-env --type space --org my-org + +# Initialize git and push +git init +git add . +git commit -m "Initial environment implementation" +git remote add origin https://huggingface.co/my-org/my-custom-env +git push -u origin main +``` + +Alternatively, use the `huggingface_hub` Python API: + +```python +from huggingface_hub import HfApi + +api = HfApi() + +# Create repository +api.create_repo("my-custom-env", repo_type="space") + +# Upload files +api.upload_folder( + folder_path="./my-env-folder", + repo_id="username/my-custom-env", + repo_type="space", +) +``` + +## Loading Environments from the Hub + +### Basic Usage + +```python +from lerobot.envs.factory import make_env + +# Load from the hub +envs_dict = make_env( + "username/my-custom-env", + n_envs=4, + trust_remote_code=True +) + +# Access the environment +suite_name = next(iter(envs_dict)) +env = envs_dict[suite_name][0] + +# Use it like any gym environment +obs, info = env.reset() +action = env.action_space.sample() +obs, reward, terminated, truncated, info = env.step(action) +``` + +### Advanced: Pinning to Specific Versions + +For reproducibility and security, pin to a specific Git revision: + +```python +# Pin to a specific branch +env = make_env("username/my-env@main", trust_remote_code=True) + +# Pin to a specific commit (recommended for papers/experiments) +env = make_env("username/my-env@abc123def456", trust_remote_code=True) + +# Pin to a tag +env = make_env("username/my-env@v1.0.0", trust_remote_code=True) +``` + +### Custom File Paths + +If your environment definition is not in `env.py`: + +```python +# Load from a custom file +env = make_env("username/my-env:custom_env.py", trust_remote_code=True) + +# Combine with version pinning +env = make_env("username/my-env@v1.0:envs/task_a.py", trust_remote_code=True) +``` + +### Async Environments + +For better performance with multiple environments: + +```python +envs_dict = make_env( + "username/my-env", + n_envs=8, + use_async_envs=True, # Use AsyncVectorEnv for parallel execution + trust_remote_code=True +) +``` + +## URL Format Reference + +The hub URL format supports several patterns: + +| Pattern | Description | Example | +| -------------------- | ------------------------------ | -------------------------------------- | +| `user/repo` | Load `env.py` from main branch | `make_env("lerobot/pusht-env")` | +| `user/repo@revision` | Load from specific revision | `make_env("lerobot/pusht-env@main")` | +| `user/repo:path` | Load custom file | `make_env("lerobot/envs:pusht.py")` | +| `user/repo@rev:path` | Revision + custom file | `make_env("lerobot/envs@v1:pusht.py")` | + +## Multi-Task Environments + +For benchmarks with multiple tasks (like LIBERO), return a nested dictionary: + +```python +def make_env(n_envs: int = 1, use_async_envs: bool = False): + env_cls = gym.vector.AsyncVectorEnv if use_async_envs else gym.vector.SyncVectorEnv + + # Return dict: {suite_name: {task_id: VectorEnv}} + return { + "suite_1": { + 0: env_cls([lambda: gym.make("Task1-v0") for _ in range(n_envs)]), + 1: env_cls([lambda: gym.make("Task2-v0") for _ in range(n_envs)]), + }, + "suite_2": { + 0: env_cls([lambda: gym.make("Task3-v0") for _ in range(n_envs)]), + } + } +``` + +## Security Considerations + + + **Important**: The `trust_remote_code=True` flag is required to execute + environment code from the Hub. This is by design for security. + + +When loading environments from the Hub: + +1. **Review the code first**: Visit the repository and inspect `env.py` before loading +2. **Pin to commits**: Use specific commit hashes for reproducibility +3. **Check dependencies**: Review `requirements.txt` for suspicious packages +4. **Use trusted sources**: Prefer official organizations or well-known researchers +5. **Sandbox if needed**: Run untrusted code in isolated environments (containers, VMs) + +Example of safe usage: + +```python +# ❌ BAD: Loading without inspection +env = make_env("random-user/untrusted-env", trust_remote_code=True) + +# ✅ GOOD: Review code, then pin to specific commit +# 1. Visit https://huggingface.co/trusted-org/verified-env +# 2. Review the env.py file +# 3. Copy the commit hash +env = make_env("trusted-org/verified-env@a1b2c3d4", trust_remote_code=True) +``` + +## Example: CartPole from the Hub + +Here's a complete example using the reference CartPole environment: + +```python +from lerobot.envs.factory import make_env +import numpy as np + +# Load the environment +envs_dict = make_env("lerobot/cartpole-env", n_envs=4, trust_remote_code=True) + +# Get the vectorized environment +suite_name = next(iter(envs_dict)) +env = envs_dict[suite_name][0] + +# Run a simple episode +obs, info = env.reset() +done = np.zeros(env.num_envs, dtype=bool) +total_reward = np.zeros(env.num_envs) + +while not done.all(): + # Random policy + action = env.action_space.sample() + obs, reward, terminated, truncated, info = env.step(action) + total_reward += reward + done = terminated | truncated + +print(f"Average reward: {total_reward.mean():.2f}") +env.close() +``` + +## Benefits of EnvHub + +### For Environment Authors + +- **Easy distribution**: No PyPI packaging required +- **Version control**: Use Git for environment versioning +- **Rapid iteration**: Push updates instantly +- **Documentation**: Hub README renders beautifully +- **Community**: Reach LeRobot users directly + +### For Researchers + +- **Quick experiments**: Load any environment in one line +- **Reproducibility**: Pin to specific commits +- **Discovery**: Browse environments on the Hub +- **No conflicts**: No need to install conflicting packages + +### For the Community + +- **Growing ecosystem**: More diverse simulation tasks +- **Standardization**: Common `make_env` API +- **Collaboration**: Fork and improve existing environments +- **Accessibility**: Lower barrier to sharing research + +## Troubleshooting + +### "Refusing to execute remote code" + +You must explicitly pass `trust_remote_code=True`: + +```python +env = make_env("user/repo", trust_remote_code=True) +``` + +### "Module X not found" + +The hub environment has dependencies you need to install: + +```bash +# Check the repo's requirements.txt and install dependencies +pip install gymnasium numpy +``` + +### "make_env not found in module" + +Your `env.py` must expose a `make_env` function: + +```python +def make_env(n_envs: int, use_async_envs: bool): + # Your implementation + pass +``` + +### Environment returns wrong type + +The `make_env` function must return: + +- A `gym.vector.VectorEnv`, or +- A single `gym.Env`, or +- A dict `{suite_name: {task_id: VectorEnv}}` + +## Best Practices + +1. **Document your environment**: Include observation/action space descriptions, reward structure, and termination conditions in your README +2. **Add requirements.txt**: List all dependencies with versions +3. **Test thoroughly**: Verify your environment works locally before pushing +4. **Use semantic versioning**: Tag releases with version numbers +5. **Add examples**: Include usage examples in your README +6. **Keep it simple**: Minimize dependencies when possible +7. **License your work**: Add a LICENSE file to clarify usage terms + +## Future Directions + +The EnvHub ecosystem enables exciting possibilities: + +- **GPU-accelerated physics**: Share Isaac Gym or Brax environments +- **Photorealistic rendering**: Distribute environments with advanced graphics +- **Multi-agent scenarios**: Complex interaction tasks +- **Real-world simulators**: Digital twins of physical setups +- **Procedural generation**: Infinite task variations +- **Domain randomization**: Pre-configured DR pipelines + +As more researchers and developers contribute, the diversity and quality of available environments will grow, benefiting the entire robotics learning community. + +## See Also + +- [Hugging Face Hub Documentation](https://huggingface.co/docs/hub/en/index) +- [Gymnasium Documentation](https://gymnasium.farama.org/index.html) +- [Example Hub Environment](https://huggingface.co/lerobot/cartpole-env) diff --git a/docs/source/envhub_leisaac.mdx b/docs/source/envhub_leisaac.mdx new file mode 100644 index 000000000..ff848d415 --- /dev/null +++ b/docs/source/envhub_leisaac.mdx @@ -0,0 +1,301 @@ +# LeIsaac × LeRobot EnvHub + +LeRobot EnvHub now supports **imitation learning in simulation** with LeIsaac. +Spin up everyday manipulation tasks, teleoperate the robot, collect demos, push them to the Hub, and train policies in LeRobot — all in one loop. + +[LeIsaac](https://github.com/LightwheelAI/leisaac) integrates with IsaacLab and the SO101 Leader/Follower setup to provide: + +- 🕹️ **Teleoperation-first workflows** for data collection +- 📦 **Built-in data conversion** ready for LeRobot training +- 🤖 **Everyday skills** like picking oranges, lifting cubes, cleaning tables, and folding cloth +- ☁️ **Ongoing upgrades** from [LightWheel](https://lightwheel.ai/): cloud simulation, EnvHub support, Sim2Real tooling, and more + +Below you’ll find the currently supported LeIsaac tasks exposed through LeRobot EnvHub. + +# Available Environments + +The following table lists all available tasks and environments in LeIsaac x LeRobot Envhub. You can also get the latest list of environments by running the following command: + +```bash +python scripts/environments/list_envs.py +``` + +| Task | Environment ID | Task Description | Related Robot | +| :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------- | +| | [LeIsaac-SO101-PickOrange-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/pick_orange/pick_orange_env_cfg.py)

[LeIsaac-SO101-PickOrange-Direct-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/pick_orange/direct/pick_orange_env.py) | Pick three oranges and put them into the plate, then reset the arm to rest state. | Single-Arm SO101 Follower | +| | [LeIsaac-SO101-LiftCube-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/lift_cube/lift_cube_env_cfg.py)

[LeIsaac-SO101-LiftCube-Direct-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/lift_cube/direct/lift_cube_env.py) | Lift the red cube up. | Single-Arm SO101 Follower | +| | [LeIsaac-SO101-CleanToyTable-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/clean_toy_table/clean_toy_table_env_cfg.py)

[LeIsaac-SO101-CleanToyTable-BiArm-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/clean_toy_table/clean_toy_table_bi_arm_env_cfg.py)

[LeIsaac-SO101-CleanToyTable-BiArm-Direct-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/clean_toy_table/direct/clean_toy_table_bi_arm_env.py) | Pick two letter e objects into the box, and reset the arm to rest state. | Single-Arm SO101 Follower

Bi-Arm SO101 Follower | +| | [LeIsaac-SO101-FoldCloth-BiArm-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/fold_cloth/fold_cloth_bi_arm_env_cfg.py)

[LeIsaac-SO101-FoldCloth-BiArm-Direct-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/fold_cloth/direct/fold_cloth_bi_arm_env.py) | Fold the cloth, and reset the arm to rest state.

_Note: Only the DirectEnv support check_success in this task._ | Bi-Arm SO101 Follower | + +# Load LeIsaac directly in LeRobot with one line of code + +> EnvHub: Share LeIsaac environments through HuggingFace + +[EnvHub](https://huggingface.co/docs/lerobot/envhub) is our reproducible environment hub, spin up a packaged simulation with one line, experiment immediately, and publish your own tasks for the community. + +LeIsaac offers EnvHub support so you can consume or share tasks with only a few commands. + +