mirror of
https://github.com/huggingface/lerobot.git
synced 2026-05-30 18:31:25 +00:00
Compare commits
3 Commits
docs/compl
...
6b647fee2c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6b647fee2c | ||
|
|
5c98e80430 | ||
|
|
f65f3f7a4a |
@@ -1,172 +0,0 @@
|
|||||||
- sections:
|
|
||||||
- local: index
|
|
||||||
title: LeRobot
|
|
||||||
- local: installation
|
|
||||||
title: Installation
|
|
||||||
- local: cheat-sheet
|
|
||||||
title: Cheat sheet
|
|
||||||
title: Get started
|
|
||||||
- sections:
|
|
||||||
- local: il_robots
|
|
||||||
title: Imitation Learning for Robots
|
|
||||||
- local: bring_your_own_policies
|
|
||||||
title: Adding a Policy
|
|
||||||
- local: integrate_hardware
|
|
||||||
title: Bring Your Own Hardware
|
|
||||||
- local: hilserl
|
|
||||||
title: Train a Robot with RL
|
|
||||||
- local: hilserl_sim
|
|
||||||
title: Train RL in Simulation
|
|
||||||
- local: multi_gpu_training
|
|
||||||
title: Multi GPU training
|
|
||||||
- local: hil_data_collection
|
|
||||||
title: Human In the Loop Data Collection
|
|
||||||
- local: peft_training
|
|
||||||
title: Training with PEFT (e.g., LoRA)
|
|
||||||
- local: rename_map
|
|
||||||
title: Using Rename Map and Empty Cameras
|
|
||||||
title: "Tutorials"
|
|
||||||
- sections:
|
|
||||||
- local: hardware_guide
|
|
||||||
title: Compute Hardware Guide
|
|
||||||
- local: torch_accelerators
|
|
||||||
title: PyTorch accelerators
|
|
||||||
title: "Compute & Hardware"
|
|
||||||
- sections:
|
|
||||||
- local: lerobot-dataset-v3
|
|
||||||
title: Using LeRobotDataset
|
|
||||||
- local: porting_datasets_v3
|
|
||||||
title: Porting Large Datasets
|
|
||||||
- local: using_dataset_tools
|
|
||||||
title: Using the Dataset Tools
|
|
||||||
- local: language_and_recipes
|
|
||||||
title: Language Columns and Recipes
|
|
||||||
- local: tools
|
|
||||||
title: Tools
|
|
||||||
- local: video_encoding_parameters
|
|
||||||
title: Video encoding parameters
|
|
||||||
- local: streaming_video_encoding
|
|
||||||
title: Streaming Video Encoding
|
|
||||||
title: "Datasets"
|
|
||||||
- sections:
|
|
||||||
- local: act
|
|
||||||
title: ACT
|
|
||||||
- local: smolvla
|
|
||||||
title: SmolVLA
|
|
||||||
- local: pi0
|
|
||||||
title: π₀ (Pi0)
|
|
||||||
- local: pi0fast
|
|
||||||
title: π₀-FAST (Pi0Fast)
|
|
||||||
- local: pi05
|
|
||||||
title: π₀.₅ (Pi05)
|
|
||||||
- local: eo1
|
|
||||||
title: EO-1
|
|
||||||
- local: groot
|
|
||||||
title: NVIDIA GR00T N1.5
|
|
||||||
- local: xvla
|
|
||||||
title: X-VLA
|
|
||||||
- local: multi_task_dit
|
|
||||||
title: Multitask DiT Policy
|
|
||||||
- local: walloss
|
|
||||||
title: WALL-OSS
|
|
||||||
title: "Policies"
|
|
||||||
- sections:
|
|
||||||
- local: sarm
|
|
||||||
title: SARM
|
|
||||||
title: "Reward Models"
|
|
||||||
- sections:
|
|
||||||
- local: inference
|
|
||||||
title: Policy Deployment (lerobot-rollout)
|
|
||||||
- local: async
|
|
||||||
title: Use Async Inference
|
|
||||||
- local: rtc
|
|
||||||
title: Real-Time Chunking (RTC)
|
|
||||||
title: "Inference"
|
|
||||||
- sections:
|
|
||||||
- local: envhub
|
|
||||||
title: Environments from the Hub
|
|
||||||
- local: envhub_leisaac
|
|
||||||
title: Control & Train Robots in Sim (LeIsaac)
|
|
||||||
title: "Simulation"
|
|
||||||
- sections:
|
|
||||||
- local: adding_benchmarks
|
|
||||||
title: Adding a New Benchmark
|
|
||||||
- local: libero
|
|
||||||
title: LIBERO
|
|
||||||
- local: libero_plus
|
|
||||||
title: LIBERO-plus
|
|
||||||
- local: metaworld
|
|
||||||
title: Meta-World
|
|
||||||
- local: robotwin
|
|
||||||
title: RoboTwin 2.0
|
|
||||||
- local: robocasa
|
|
||||||
title: RoboCasa365
|
|
||||||
- local: robocerebra
|
|
||||||
title: RoboCerebra
|
|
||||||
- local: robomme
|
|
||||||
title: RoboMME
|
|
||||||
- local: envhub_isaaclab_arena
|
|
||||||
title: NVIDIA IsaacLab Arena Environments
|
|
||||||
- local: vlabench
|
|
||||||
title: VLABench
|
|
||||||
title: "Benchmarks"
|
|
||||||
- sections:
|
|
||||||
- local: introduction_processors
|
|
||||||
title: Introduction to Robot Processors
|
|
||||||
- local: debug_processor_pipeline
|
|
||||||
title: Debug your processor pipeline
|
|
||||||
- local: implement_your_own_processor
|
|
||||||
title: Implement your own processor
|
|
||||||
- local: processors_robots_teleop
|
|
||||||
title: Processors for Robots and Teleoperators
|
|
||||||
- local: env_processor
|
|
||||||
title: Environment Processors
|
|
||||||
- local: action_representations
|
|
||||||
title: Action Representations
|
|
||||||
title: "Robot Processors"
|
|
||||||
- sections:
|
|
||||||
- local: so101
|
|
||||||
title: SO-101
|
|
||||||
- local: so100
|
|
||||||
title: SO-100
|
|
||||||
- local: koch
|
|
||||||
title: Koch v1.1
|
|
||||||
- local: lekiwi
|
|
||||||
title: LeKiwi
|
|
||||||
- local: hope_jr
|
|
||||||
title: Hope Jr
|
|
||||||
- local: reachy2
|
|
||||||
title: Reachy 2
|
|
||||||
- local: unitree_g1
|
|
||||||
title: Unitree G1
|
|
||||||
- local: earthrover_mini_plus
|
|
||||||
title: Earth Rover Mini
|
|
||||||
- local: omx
|
|
||||||
title: OMX
|
|
||||||
- local: openarm
|
|
||||||
title: OpenArm
|
|
||||||
- local: rebot_b601
|
|
||||||
title: reBot B601-DM
|
|
||||||
title: "Robots"
|
|
||||||
- sections:
|
|
||||||
- local: phone_teleop
|
|
||||||
title: Phone
|
|
||||||
title: "Teleoperators"
|
|
||||||
- sections:
|
|
||||||
- local: cameras
|
|
||||||
title: Cameras
|
|
||||||
title: "Sensors"
|
|
||||||
- sections:
|
|
||||||
- local: notebooks
|
|
||||||
title: Notebooks
|
|
||||||
- local: feetech
|
|
||||||
title: Updating Feetech Firmware
|
|
||||||
- local: damiao
|
|
||||||
title: Damiao Motors and CAN Bus
|
|
||||||
title: "Resources"
|
|
||||||
- sections:
|
|
||||||
- local: contributing
|
|
||||||
title: Contribute to LeRobot
|
|
||||||
- local: backwardcomp
|
|
||||||
title: Backward compatibility
|
|
||||||
title: "About"
|
|
||||||
@@ -1,214 +1,172 @@
|
|||||||
# LeRobot documentation table of contents
|
|
||||||
#
|
|
||||||
# Ordering principle: gentle onboarding first, advanced/custom work last.
|
|
||||||
# Within each top-level section the same rule applies — concept/overview pages
|
|
||||||
# before reference/per-item pages.
|
|
||||||
#
|
|
||||||
# Pages marked "NEW (to create)" do not yet exist as .mdx files; they are
|
|
||||||
# placeholders for the redesign and must be authored before the docs build.
|
|
||||||
|
|
||||||
- sections:
|
- sections:
|
||||||
- local: index
|
- local: index
|
||||||
title: 🤗 LeRobot
|
title: LeRobot
|
||||||
- local: quickstart # NEW (to create) — 15-min zero-to-trained-ACT path
|
|
||||||
title: Quickstart
|
|
||||||
- local: installation
|
- local: installation
|
||||||
title: Installation
|
title: Installation
|
||||||
- local: core_concepts # NEW (to create) — datasets, policies, processors, robots, envs in one mental model
|
|
||||||
title: Core concepts
|
|
||||||
- local: cheat-sheet
|
- local: cheat-sheet
|
||||||
title: Command cheat sheet
|
title: Cheat sheet
|
||||||
title: Get started
|
title: Get started
|
||||||
|
|
||||||
- sections:
|
- sections:
|
||||||
- local: il_robots
|
- local: il_robots
|
||||||
title: Imitation learning end-to-end
|
title: Imitation Learning for Robots
|
||||||
|
- local: bring_your_own_policies
|
||||||
|
title: Adding a Policy
|
||||||
|
- local: integrate_hardware
|
||||||
|
title: Bring Your Own Hardware
|
||||||
|
- local: hilserl
|
||||||
|
title: Train a Robot with RL
|
||||||
|
- local: hilserl_sim
|
||||||
|
title: Train RL in Simulation
|
||||||
|
- local: multi_gpu_training
|
||||||
|
title: Multi GPU training
|
||||||
- local: hil_data_collection
|
- local: hil_data_collection
|
||||||
title: Human-in-the-loop data collection
|
title: Human In the Loop Data Collection
|
||||||
- local: inference
|
- local: peft_training
|
||||||
title: Deploying a trained policy
|
title: Training with PEFT (e.g., LoRA)
|
||||||
- local: rename_map
|
- local: rename_map
|
||||||
title: Matching dataset keys to a policy (rename map)
|
title: Using Rename Map and Empty Cameras
|
||||||
title: Your first project
|
title: "Tutorials"
|
||||||
|
|
||||||
- sections:
|
- sections:
|
||||||
- local: hardware_guide
|
- local: hardware_guide
|
||||||
title: Compute hardware guide
|
title: Compute Hardware Guide
|
||||||
- local: torch_accelerators
|
- local: torch_accelerators
|
||||||
title: PyTorch accelerators
|
title: PyTorch accelerators
|
||||||
- local: multi_gpu_training
|
title: "Compute & Hardware"
|
||||||
title: Multi-GPU training
|
|
||||||
- local: peft_training
|
|
||||||
title: Parameter-efficient fine-tuning (LoRA)
|
|
||||||
title: Training
|
|
||||||
|
|
||||||
- sections:
|
- sections:
|
||||||
- local: lerobot-dataset-v3
|
- local: lerobot-dataset-v3
|
||||||
title: Using LeRobotDataset
|
title: Using LeRobotDataset
|
||||||
|
- local: porting_datasets_v3
|
||||||
|
title: Porting Large Datasets
|
||||||
- local: using_dataset_tools
|
- local: using_dataset_tools
|
||||||
title: Dataset tools
|
title: Using the Dataset Tools
|
||||||
- local: language_and_recipes
|
- local: language_and_recipes
|
||||||
title: Language columns & recipes
|
title: Language Columns and Recipes
|
||||||
- local: tools
|
- local: tools
|
||||||
title: Tool calls in datasets
|
title: Tools
|
||||||
- local: video_encoding_parameters
|
- local: video_encoding_parameters
|
||||||
title: Video encoding parameters
|
title: Video encoding parameters
|
||||||
- local: streaming_video_encoding
|
- local: streaming_video_encoding
|
||||||
title: Streaming video encoding
|
title: Streaming Video Encoding
|
||||||
- local: porting_datasets_v3
|
title: "Datasets"
|
||||||
title: Porting datasets to v3
|
|
||||||
title: Datasets
|
|
||||||
|
|
||||||
- sections:
|
- sections:
|
||||||
- local: policies_overview # NEW (to create) — concept hub + "choose a policy" decision guide
|
- local: act
|
||||||
title: Choosing a policy
|
title: ACT
|
||||||
- sections:
|
- local: smolvla
|
||||||
- local: act
|
title: SmolVLA
|
||||||
title: ACT
|
- local: pi0
|
||||||
- local: smolvla
|
title: π₀ (Pi0)
|
||||||
title: SmolVLA
|
- local: pi0fast
|
||||||
- local: pi0
|
title: π₀-FAST (Pi0Fast)
|
||||||
title: π₀ (Pi0)
|
- local: pi05
|
||||||
- local: pi0fast
|
title: π₀.₅ (Pi05)
|
||||||
title: π₀-FAST
|
- local: eo1
|
||||||
- local: pi05
|
title: EO-1
|
||||||
title: π₀.₅ (Pi05)
|
- local: groot
|
||||||
- local: eo1
|
title: NVIDIA GR00T N1.5
|
||||||
title: EO-1
|
- local: xvla
|
||||||
- local: groot
|
title: X-VLA
|
||||||
title: NVIDIA GR00T N1.5
|
- local: multi_task_dit
|
||||||
- local: xvla
|
title: Multitask DiT Policy
|
||||||
title: X-VLA
|
- local: walloss
|
||||||
- local: walloss
|
title: WALL-OSS
|
||||||
title: WALL-OSS
|
title: "Policies"
|
||||||
- local: multi_task_dit
|
|
||||||
title: Multitask DiT
|
|
||||||
title: Policy reference
|
|
||||||
title: Policies
|
|
||||||
|
|
||||||
- sections:
|
- sections:
|
||||||
- local: async
|
|
||||||
title: Async inference
|
|
||||||
- local: rtc
|
|
||||||
title: Real-time chunking (RTC)
|
|
||||||
title: Real-time deployment
|
|
||||||
|
|
||||||
- sections:
|
|
||||||
- local: hilserl
|
|
||||||
title: Train a robot with RL (HIL-SERL)
|
|
||||||
- local: hilserl_sim
|
|
||||||
title: Train RL in simulation
|
|
||||||
- local: sarm
|
- local: sarm
|
||||||
title: SARM reward model
|
title: SARM
|
||||||
title: Reinforcement learning
|
title: "Reward Models"
|
||||||
|
- sections:
|
||||||
|
- local: inference
|
||||||
|
title: Policy Deployment (lerobot-rollout)
|
||||||
|
- local: async
|
||||||
|
title: Use Async Inference
|
||||||
|
- local: rtc
|
||||||
|
title: Real-Time Chunking (RTC)
|
||||||
|
title: "Inference"
|
||||||
- sections:
|
- sections:
|
||||||
- local: envhub
|
- local: envhub
|
||||||
title: Environments from the Hub
|
title: Environments from the Hub
|
||||||
- local: envhub_leisaac
|
- local: envhub_leisaac
|
||||||
title: LeIsaac — control & train in sim
|
title: Control & Train Robots in Sim (LeIsaac)
|
||||||
|
title: "Simulation"
|
||||||
|
- sections:
|
||||||
|
- local: adding_benchmarks
|
||||||
|
title: Adding a New Benchmark
|
||||||
|
- local: libero
|
||||||
|
title: LIBERO
|
||||||
|
- local: libero_plus
|
||||||
|
title: LIBERO-plus
|
||||||
|
- local: metaworld
|
||||||
|
title: Meta-World
|
||||||
|
- local: robotwin
|
||||||
|
title: RoboTwin 2.0
|
||||||
|
- local: robocasa
|
||||||
|
title: RoboCasa365
|
||||||
|
- local: robocerebra
|
||||||
|
title: RoboCerebra
|
||||||
|
- local: robomme
|
||||||
|
title: RoboMME
|
||||||
- local: envhub_isaaclab_arena
|
- local: envhub_isaaclab_arena
|
||||||
title: NVIDIA IsaacLab Arena environments
|
title: NVIDIA IsaacLab Arena Environments
|
||||||
- sections:
|
- local: vlabench
|
||||||
- local: libero
|
title: VLABench
|
||||||
title: LIBERO
|
title: "Benchmarks"
|
||||||
- local: libero_plus
|
|
||||||
title: LIBERO-plus
|
|
||||||
- local: metaworld
|
|
||||||
title: Meta-World
|
|
||||||
- local: robotwin
|
|
||||||
title: RoboTwin 2.0
|
|
||||||
- local: robocasa
|
|
||||||
title: RoboCasa365
|
|
||||||
- local: robocerebra
|
|
||||||
title: RoboCerebra
|
|
||||||
- local: robomme
|
|
||||||
title: RoboMME
|
|
||||||
- local: vlabench
|
|
||||||
title: VLABench
|
|
||||||
title: Benchmark suites
|
|
||||||
title: Simulation & benchmarks
|
|
||||||
|
|
||||||
- sections:
|
- sections:
|
||||||
- local: introduction_processors
|
- local: introduction_processors
|
||||||
title: Introduction to processors
|
title: Introduction to Robot Processors
|
||||||
- local: processors_robots_teleop
|
|
||||||
title: Processors for robots & teleoperators
|
|
||||||
- local: env_processor
|
|
||||||
title: Environment processors
|
|
||||||
- local: action_representations
|
|
||||||
title: Action representations
|
|
||||||
- local: debug_processor_pipeline
|
- local: debug_processor_pipeline
|
||||||
title: Debugging a pipeline
|
title: Debug your processor pipeline
|
||||||
- local: implement_your_own_processor
|
- local: implement_your_own_processor
|
||||||
title: Implementing your own processor
|
title: Implement your own processor
|
||||||
title: Processors
|
- local: processors_robots_teleop
|
||||||
|
title: Processors for Robots and Teleoperators
|
||||||
|
- local: env_processor
|
||||||
|
title: Environment Processors
|
||||||
|
- local: action_representations
|
||||||
|
title: Action Representations
|
||||||
|
title: "Robot Processors"
|
||||||
- sections:
|
- sections:
|
||||||
- sections:
|
- local: so101
|
||||||
- local: so101
|
title: SO-101
|
||||||
title: SO-101
|
- local: so100
|
||||||
- local: so100
|
title: SO-100
|
||||||
title: SO-100
|
- local: koch
|
||||||
- local: koch
|
title: Koch v1.1
|
||||||
title: Koch v1.1
|
- local: lekiwi
|
||||||
- local: omx
|
title: LeKiwi
|
||||||
title: OMX
|
- local: hope_jr
|
||||||
- local: openarm
|
title: Hope Jr
|
||||||
title: OpenArm
|
- local: reachy2
|
||||||
title: Low-cost arms
|
title: Reachy 2
|
||||||
- sections:
|
- local: unitree_g1
|
||||||
- local: lekiwi
|
title: Unitree G1
|
||||||
title: LeKiwi
|
- local: earthrover_mini_plus
|
||||||
- local: earthrover_mini_plus
|
title: Earth Rover Mini
|
||||||
title: Earth Rover Mini
|
- local: omx
|
||||||
title: Mobile platforms
|
title: OMX
|
||||||
- sections:
|
- local: openarm
|
||||||
- local: hope_jr
|
title: OpenArm
|
||||||
title: Hope Jr
|
- local: rebot_b601
|
||||||
- local: reachy2
|
title: reBot B601-DM
|
||||||
title: Reachy 2
|
title: "Robots"
|
||||||
- local: unitree_g1
|
- sections:
|
||||||
title: Unitree G1
|
- local: phone_teleop
|
||||||
title: Bimanual & humanoid
|
title: Phone
|
||||||
- sections:
|
title: "Teleoperators"
|
||||||
- local: rebot_b601
|
|
||||||
title: reBot B601-DM
|
|
||||||
title: Research & industrial
|
|
||||||
title: Supported robots
|
|
||||||
|
|
||||||
- sections:
|
- sections:
|
||||||
- local: cameras
|
- local: cameras
|
||||||
title: Cameras
|
title: Cameras
|
||||||
- local: phone_teleop
|
title: "Sensors"
|
||||||
title: Phone teleoperation
|
|
||||||
- local: feetech
|
|
||||||
title: Feetech firmware update
|
|
||||||
- local: damiao
|
|
||||||
title: Damiao motors & CAN bus
|
|
||||||
title: Sensors, teleop & motors
|
|
||||||
|
|
||||||
- sections:
|
- sections:
|
||||||
- local: integrate_hardware
|
|
||||||
title: Bring your own hardware
|
|
||||||
- local: bring_your_own_policies
|
|
||||||
title: Add a new policy
|
|
||||||
- local: adding_benchmarks
|
|
||||||
title: Add a new benchmark
|
|
||||||
title: Extend LeRobot
|
|
||||||
|
|
||||||
- sections:
|
|
||||||
- local: troubleshooting # NEW (to create) — common errors: USB, calibration drift, CUDA OOM, video decoding…
|
|
||||||
title: Troubleshooting & FAQ
|
|
||||||
- local: glossary # NEW (to create) — episode, action chunk, leader/follower, teleop, processor…
|
|
||||||
title: Glossary
|
|
||||||
- local: notebooks
|
- local: notebooks
|
||||||
title: Example notebooks
|
title: Notebooks
|
||||||
- local: backwardcomp
|
- local: feetech
|
||||||
title: Backward compatibility
|
title: Updating Feetech Firmware
|
||||||
title: Reference
|
- local: damiao
|
||||||
|
title: Damiao Motors and CAN Bus
|
||||||
|
title: "Resources"
|
||||||
- sections:
|
- sections:
|
||||||
- local: contributing
|
- local: contributing
|
||||||
title: Contributing to LeRobot
|
title: Contribute to LeRobot
|
||||||
title: About
|
- local: backwardcomp
|
||||||
|
title: Backward compatibility
|
||||||
|
title: "About"
|
||||||
|
|||||||
@@ -1,219 +0,0 @@
|
|||||||
# Quickstart
|
|
||||||
|
|
||||||
This is the **shortest path** from an unboxed SO-101 to a policy that drives your own robot. Every step is copy-paste; replace the **`<placeholders>`** with the values for your setup.
|
|
||||||
|
|
||||||
By the end you will have:
|
|
||||||
|
|
||||||
- A calibrated SO-101 leader + follower pair.
|
|
||||||
- A dataset of 30 episodes pushed to the Hugging Face Hub.
|
|
||||||
- A trained ACT policy (~20k steps) running on your robot via `lerobot-rollout`.
|
|
||||||
|
|
||||||
> [!NOTE]
|
|
||||||
> **How long will this take?**
|
|
||||||
> Recording 30 episodes is roughly 30–60 minutes of teleoperation. Training ACT for 20k steps takes ~1.5h on an A100, a few hours on a laptop RTX 3060, longer on Apple Silicon (`mps`). The commands themselves are quick — most of the wall-clock is data collection and training.
|
|
||||||
|
|
||||||
> [!TIP]
|
|
||||||
> If you only want to **understand the codebase** or **train on an existing dataset without hardware**, this page isn't for you. Read [Core concepts](./core_concepts) first, then jump to [Imitation learning end-to-end](./il_robots).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Before you start
|
|
||||||
|
|
||||||
You need:
|
|
||||||
|
|
||||||
- An **assembled SO-101 leader + follower pair**. If your robot is not assembled yet, follow the [SO-101 assembly guide](./so101) and come back here.
|
|
||||||
- **One or two cameras** (USB webcam works fine).
|
|
||||||
- A **CUDA GPU with ≥ 6 GB VRAM** (ACT is light — a laptop RTX 3060 works). Apple Silicon (`mps`) and CPU are supported but slower. See the [compute hardware guide](./hardware_guide) for sizing.
|
|
||||||
- A **Hugging Face account** — datasets and the trained policy will be pushed to your Hub.
|
|
||||||
|
|
||||||
If any of the above is missing, fix it first; the rest of the page assumes it.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Step 1 — Install LeRobot
|
|
||||||
|
|
||||||
Follow the full [Installation Guide](./installation) for environment setup, then add the SO-101 motor stack and log in to the Hub:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pip install 'lerobot[feetech]'
|
|
||||||
git lfs install && git lfs pull
|
|
||||||
hf auth login # paste a token from https://huggingface.co/settings/tokens
|
|
||||||
```
|
|
||||||
|
|
||||||
Sanity check — the CLI entry points should be available:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
lerobot-find-port --help
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Step 2 — Identify USB ports and motor IDs
|
|
||||||
|
|
||||||
Plug **only the follower arm** in (USB + power) and run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
lerobot-find-port
|
|
||||||
```
|
|
||||||
|
|
||||||
When prompted, unplug it and press Enter. Note the printed port — that's your `<FOLLOWER_PORT>`. Repeat with only the **leader arm** plugged in to get `<LEADER_PORT>`.
|
|
||||||
|
|
||||||
> [!TIP]
|
|
||||||
> On Linux, USB ports look like `/dev/ttyACM0`; on macOS like `/dev/tty.usbmodem...`. On Linux you may need `sudo chmod 666 /dev/ttyACM0` to grant access.
|
|
||||||
|
|
||||||
If your motors are brand-new (or repurposed), set their IDs and baudrate **once per arm**:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
lerobot-setup-motors --robot.type=so101_follower --robot.port=<FOLLOWER_PORT>
|
|
||||||
lerobot-setup-motors --teleop.type=so101_leader --teleop.port=<LEADER_PORT>
|
|
||||||
```
|
|
||||||
|
|
||||||
The script walks you through connecting motors one at a time. Full details: [SO-101 → Configure the motors](./so101#configure-the-motors).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Step 3 — Calibrate
|
|
||||||
|
|
||||||
Center every joint roughly in the middle of its range, then run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
lerobot-calibrate \
|
|
||||||
--robot.type=so101_follower \
|
|
||||||
--robot.port=<FOLLOWER_PORT> \
|
|
||||||
--robot.id=my_follower
|
|
||||||
|
|
||||||
lerobot-calibrate \
|
|
||||||
--teleop.type=so101_leader \
|
|
||||||
--teleop.port=<LEADER_PORT> \
|
|
||||||
--teleop.id=my_leader
|
|
||||||
```
|
|
||||||
|
|
||||||
After pressing Enter, sweep each joint through its full range of motion, then press Enter again to finish.
|
|
||||||
|
|
||||||
> [!WARNING]
|
|
||||||
> The `--robot.id` / `--teleop.id` values (`my_follower`, `my_leader`) become the **calibration keys**. Reuse the same IDs in every later command — that's how LeRobot finds the calibration on disk.
|
|
||||||
|
|
||||||
Watch the [calibration video](./so101#calibrate) if anything is unclear.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Step 4 — Teleoperate (sanity check, no recording)
|
|
||||||
|
|
||||||
Before recording anything, confirm the leader drives the follower correctly:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
lerobot-teleoperate \
|
|
||||||
--robot.type=so101_follower \
|
|
||||||
--robot.port=<FOLLOWER_PORT> \
|
|
||||||
--robot.id=my_follower \
|
|
||||||
--robot.cameras="{ top: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30} }" \
|
|
||||||
--teleop.type=so101_leader \
|
|
||||||
--teleop.port=<LEADER_PORT> \
|
|
||||||
--teleop.id=my_leader \
|
|
||||||
--display_data=true
|
|
||||||
```
|
|
||||||
|
|
||||||
A Rerun window should open showing the camera feed and joint angles. Move the leader — the follower should mirror it in real time. If it doesn't, see [Troubleshooting & FAQ](./troubleshooting).
|
|
||||||
|
|
||||||
Don't know which camera index is which? Run `lerobot-find-cameras` — it saves a frame from each detected camera so you can pick the right one.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Step 5 — Record a dataset (30 episodes)
|
|
||||||
|
|
||||||
Now record demonstrations. Pick a short, repeatable task (e.g. *"put the red brick in the bowl"*). The dataset is pushed to the Hub under your username:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
export HF_USER=<your-hf-username>
|
|
||||||
|
|
||||||
lerobot-record \
|
|
||||||
--robot.type=so101_follower \
|
|
||||||
--robot.port=<FOLLOWER_PORT> \
|
|
||||||
--robot.id=my_follower \
|
|
||||||
--robot.cameras="{ top: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}, wrist: {type: opencv, index_or_path: 1, width: 640, height: 480, fps: 30} }" \
|
|
||||||
--teleop.type=so101_leader \
|
|
||||||
--teleop.port=<LEADER_PORT> \
|
|
||||||
--teleop.id=my_leader \
|
|
||||||
--dataset.repo_id=${HF_USER}/so101_quickstart \
|
|
||||||
--dataset.num_episodes=30 \
|
|
||||||
--dataset.single_task="Put the red brick in the bowl" \
|
|
||||||
--dataset.streaming_encoding=true \
|
|
||||||
--display_data=true
|
|
||||||
```
|
|
||||||
|
|
||||||
**Keyboard controls during recording:**
|
|
||||||
|
|
||||||
- **`→` (Right Arrow)** — save the current episode and move to the next.
|
|
||||||
- **`←` (Left Arrow)** — discard the current episode and retry.
|
|
||||||
- **`Esc`** — stop, encode videos, and upload to the Hub.
|
|
||||||
|
|
||||||
> [!TIP]
|
|
||||||
> **Quality beats quantity.** 30 clean, varied episodes (different brick positions, lighting, camera shake) train a much better policy than 100 identical ones. Move the object around. Vary your speed slightly.
|
|
||||||
|
|
||||||
When you're done, your dataset lives at `https://huggingface.co/datasets/${HF_USER}/so101_quickstart`. You can preview it in the browser. For deeper recording options (resume, multiple tasks, custom processors), see [Imitation learning end-to-end → Record](./il_robots#record-a-dataset).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Step 6 — Train ACT
|
|
||||||
|
|
||||||
ACT (Action Chunking Transformer) is the right default for a first run — small, fast, and works well on 30 episodes.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
lerobot-train \
|
|
||||||
--dataset.repo_id=${HF_USER}/so101_quickstart \
|
|
||||||
--policy.type=act \
|
|
||||||
--output_dir=outputs/train/act_so101_quickstart \
|
|
||||||
--job_name=act_so101_quickstart \
|
|
||||||
--policy.device=cuda \
|
|
||||||
--policy.repo_id=${HF_USER}/act_so101_quickstart \
|
|
||||||
--steps=20000 \
|
|
||||||
--wandb.enable=true
|
|
||||||
```
|
|
||||||
|
|
||||||
A few notes:
|
|
||||||
|
|
||||||
- Replace `--policy.device=cuda` with `mps` on Apple Silicon, or `cpu` if you have no GPU (very slow — not recommended for a real run).
|
|
||||||
- `--wandb.enable=true` is optional. If you use it, run `wandb login` first. Otherwise drop the flag.
|
|
||||||
- Checkpoints land in `outputs/train/act_so101_quickstart/checkpoints/`. The final model is also pushed to the Hub at the `--policy.repo_id` you specified.
|
|
||||||
- To resume from an interruption: `lerobot-train --config_path=outputs/train/act_so101_quickstart/checkpoints/last/pretrained_model/train_config.json --resume=true`.
|
|
||||||
|
|
||||||
> [!TIP]
|
|
||||||
> **No GPU locally?** Train on Google Colab using the [ACT notebook](./notebooks#training-act), or rent a GPU via [Hugging Face Jobs](./il_robots#train-using-hugging-face-jobs) — pay-as-you-go, no setup.
|
|
||||||
|
|
||||||
For why ACT is the default and when to switch to SmolVLA, Pi0, or another policy, see [Choosing a policy](./policies_overview).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Step 7 — Run your policy on the robot
|
|
||||||
|
|
||||||
Deploy with `lerobot-rollout`. **Use the same camera layout you used while recording** — keys and resolutions must match.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
lerobot-rollout \
|
|
||||||
--strategy.type=base \
|
|
||||||
--policy.path=${HF_USER}/act_so101_quickstart \
|
|
||||||
--robot.type=so101_follower \
|
|
||||||
--robot.port=<FOLLOWER_PORT> \
|
|
||||||
--robot.id=my_follower \
|
|
||||||
--robot.cameras="{ top: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}, wrist: {type: opencv, index_or_path: 1, width: 640, height: 480, fps: 30} }" \
|
|
||||||
--task="Put the red brick in the bowl" \
|
|
||||||
--duration=60
|
|
||||||
```
|
|
||||||
|
|
||||||
`--duration` is in seconds — leave it off to run until you stop the script. You should see the follower arm move on its own, attempting the task.
|
|
||||||
|
|
||||||
If observations from the robot use different keys than the policy expects, you'll need a [rename map](./rename_map). If latency matters, look at [async inference](./async) and [real-time chunking](./rtc).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## You're done 🎉
|
|
||||||
|
|
||||||
You now have a working IL pipeline end-to-end. From here, the natural next steps are:
|
|
||||||
|
|
||||||
- **Improve the policy** — record more diverse episodes, train longer, or try a stronger model. See [Choosing a policy](./policies_overview).
|
|
||||||
- **Go deeper on imitation learning** — [Imitation learning end-to-end](./il_robots) covers multi-camera setups, multi-task datasets, episode replay, evaluation, and Hugging Face Jobs.
|
|
||||||
- **Try RL with a human in the loop** — [HIL-SERL](./hilserl) trains a policy that improves while you correct it.
|
|
||||||
- **Use a different robot** — see [Supported robots](./so101) for low-cost arms, mobile platforms, bimanual, and humanoid.
|
|
||||||
- **Build something new** — [Bring your own hardware](./integrate_hardware) and [Add a new policy](./bring_your_own_policies).
|
|
||||||
|
|
||||||
Stuck on something? Check [Troubleshooting & FAQ](./troubleshooting), or ask on [Discord](https://discord.gg/s3KuuzsPFb).
|
|
||||||
@@ -255,8 +255,7 @@ def extract_path_fields_from_config(config_path: str, path_fields: list[str]) ->
|
|||||||
remaining = config_data[field]
|
remaining = config_data[field]
|
||||||
if remaining:
|
if remaining:
|
||||||
_config_yaml_overrides[field] = _flatten_to_cli_args(remaining)
|
_config_yaml_overrides[field] = _flatten_to_cli_args(remaining)
|
||||||
else:
|
del config_data[field]
|
||||||
del config_data[field]
|
|
||||||
modified = True
|
modified = True
|
||||||
|
|
||||||
if not modified:
|
if not modified:
|
||||||
@@ -311,7 +310,13 @@ def wrap(config_path: Path | None = None) -> Callable[[F], F]:
|
|||||||
cli_args = filter_arg("config_path", cli_args)
|
cli_args = filter_arg("config_path", cli_args)
|
||||||
cfg = argtype.from_pretrained(config_path_cli, cli_args=cli_args)
|
cfg = argtype.from_pretrained(config_path_cli, cli_args=cli_args)
|
||||||
else:
|
else:
|
||||||
cfg = draccus.parse(config_class=argtype, config_path=config_path, args=cli_args)
|
if config_path_cli:
|
||||||
|
cli_args = filter_arg("config_path", cli_args)
|
||||||
|
cfg = draccus.parse(
|
||||||
|
config_class=argtype,
|
||||||
|
config_path=config_path_cli or config_path,
|
||||||
|
args=cli_args,
|
||||||
|
)
|
||||||
response = fn(cfg, *args, **kwargs)
|
response = fn(cfg, *args, **kwargs)
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|||||||
@@ -60,6 +60,7 @@ class Eagle25VLPreTrainedModel(PreTrainedModel):
|
|||||||
"SiglipEncoderLayer",
|
"SiglipEncoderLayer",
|
||||||
]
|
]
|
||||||
_skip_keys_device_placement = "past_key_values"
|
_skip_keys_device_placement = "past_key_values"
|
||||||
|
_supports_flash_attn = True
|
||||||
_supports_flash_attn_2 = True
|
_supports_flash_attn_2 = True
|
||||||
_supports_cache_class = True
|
_supports_cache_class = True
|
||||||
_supports_static_cache = True
|
_supports_static_cache = True
|
||||||
|
|||||||
@@ -124,7 +124,6 @@ class Eagle25VLProcessor(ProcessorMixin):
|
|||||||
"videos_kwargs",
|
"videos_kwargs",
|
||||||
"text_kwargs",
|
"text_kwargs",
|
||||||
]
|
]
|
||||||
image_processor_class = "AutoImageProcessor"
|
|
||||||
tokenizer_class = "AutoTokenizer"
|
tokenizer_class = "AutoTokenizer"
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
|
|||||||
@@ -206,7 +206,11 @@ def _build_eagle_processor(tokenizer_assets_repo: str = DEFAULT_TOKENIZER_ASSETS
|
|||||||
"Vendor files are copied during model creation. Create the policy/model first, "
|
"Vendor files are copied during model creation. Create the policy/model first, "
|
||||||
"or call ensure_eagle_cache_ready() before building processors."
|
"or call ensure_eagle_cache_ready() before building processors."
|
||||||
)
|
)
|
||||||
proc = AutoProcessor.from_pretrained(str(cache_dir), trust_remote_code=True, use_fast=True)
|
proc = AutoProcessor.from_pretrained(
|
||||||
|
str(cache_dir),
|
||||||
|
trust_remote_code=True,
|
||||||
|
fix_mistral_regex=False,
|
||||||
|
)
|
||||||
proc.tokenizer.padding_side = "left"
|
proc.tokenizer.padding_side = "left"
|
||||||
return proc
|
return proc
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
"""Tests for policy.path support in YAML config files (issue #2957)."""
|
"""Tests for policy.path support in YAML config files (issue #2957)."""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
|
from lerobot.configs import parser
|
||||||
from lerobot.configs.parser import (
|
from lerobot.configs.parser import (
|
||||||
_config_path_args,
|
_config_path_args,
|
||||||
_config_yaml_overrides,
|
_config_yaml_overrides,
|
||||||
@@ -16,7 +20,8 @@ from lerobot.configs.parser import (
|
|||||||
|
|
||||||
|
|
||||||
def test_extract_path_fields_from_yaml():
|
def test_extract_path_fields_from_yaml():
|
||||||
"""Test that policy.path is extracted from a YAML config and removed."""
|
"""Test that policy.path is extracted from a YAML config and the policy block
|
||||||
|
is removed entirely (siblings are captured separately as cli_overrides)."""
|
||||||
config = {
|
config = {
|
||||||
"dataset": {"repo_id": "lerobot/pusht"},
|
"dataset": {"repo_id": "lerobot/pusht"},
|
||||||
"policy": {"type": "smolvla", "path": "lerobot/smolvla_base", "push_to_hub": False},
|
"policy": {"type": "smolvla", "path": "lerobot/smolvla_base", "push_to_hub": False},
|
||||||
@@ -26,26 +31,33 @@ def test_extract_path_fields_from_yaml():
|
|||||||
config_path = f.name
|
config_path = f.name
|
||||||
|
|
||||||
_config_path_args.clear()
|
_config_path_args.clear()
|
||||||
|
_config_yaml_overrides.clear()
|
||||||
cleaned_path = extract_path_fields_from_config(config_path, ["policy"])
|
cleaned_path = extract_path_fields_from_config(config_path, ["policy"])
|
||||||
|
|
||||||
# Path should be extracted and stored
|
# Path should be extracted and stored
|
||||||
assert _config_path_args["policy"] == "lerobot/smolvla_base"
|
assert _config_path_args["policy"] == "lerobot/smolvla_base"
|
||||||
|
|
||||||
# Cleaned config should not have the path field
|
# Cleaned config should not have the policy block at all -- draccus must not
|
||||||
|
# try to decode it as PreTrainedConfig; the actual config comes from
|
||||||
|
# from_pretrained(path) with the captured overrides applied on top.
|
||||||
with open(cleaned_path) as f:
|
with open(cleaned_path) as f:
|
||||||
cleaned = yaml.safe_load(f)
|
cleaned = yaml.safe_load(f)
|
||||||
assert "path" not in cleaned["policy"]
|
assert "policy" not in cleaned
|
||||||
assert cleaned["policy"]["type"] == "smolvla"
|
|
||||||
assert cleaned["policy"]["push_to_hub"] is False
|
|
||||||
|
|
||||||
# Original dataset should be untouched
|
# Original dataset should be untouched
|
||||||
assert cleaned["dataset"]["repo_id"] == "lerobot/pusht"
|
assert cleaned["dataset"]["repo_id"] == "lerobot/pusht"
|
||||||
|
|
||||||
|
# Sibling overrides (excluding type/path) captured for from_pretrained.
|
||||||
|
overrides = get_yaml_overrides("policy")
|
||||||
|
assert any("push_to_hub=false" in o for o in overrides)
|
||||||
|
|
||||||
_config_path_args.clear()
|
_config_path_args.clear()
|
||||||
|
_config_yaml_overrides.clear()
|
||||||
|
|
||||||
|
|
||||||
def test_extract_path_fields_from_json():
|
def test_extract_path_fields_from_json():
|
||||||
"""Test that policy.path is extracted from a JSON config."""
|
"""Test that policy.path is extracted from a JSON config and the policy
|
||||||
|
block is removed entirely."""
|
||||||
config = {
|
config = {
|
||||||
"policy": {"type": "act", "path": "some/local/path"},
|
"policy": {"type": "act", "path": "some/local/path"},
|
||||||
}
|
}
|
||||||
@@ -54,15 +66,17 @@ def test_extract_path_fields_from_json():
|
|||||||
config_path = f.name
|
config_path = f.name
|
||||||
|
|
||||||
_config_path_args.clear()
|
_config_path_args.clear()
|
||||||
|
_config_yaml_overrides.clear()
|
||||||
cleaned_path = extract_path_fields_from_config(config_path, ["policy"])
|
cleaned_path = extract_path_fields_from_config(config_path, ["policy"])
|
||||||
|
|
||||||
assert _config_path_args["policy"] == "some/local/path"
|
assert _config_path_args["policy"] == "some/local/path"
|
||||||
|
|
||||||
with open(cleaned_path) as f:
|
with open(cleaned_path) as f:
|
||||||
cleaned = json.load(f)
|
cleaned = json.load(f)
|
||||||
assert "path" not in cleaned["policy"]
|
assert "policy" not in cleaned
|
||||||
|
|
||||||
_config_path_args.clear()
|
_config_path_args.clear()
|
||||||
|
_config_yaml_overrides.clear()
|
||||||
|
|
||||||
|
|
||||||
def test_extract_no_path_returns_original():
|
def test_extract_no_path_returns_original():
|
||||||
@@ -216,3 +230,91 @@ def test_flatten_nested_with_bools():
|
|||||||
args = _flatten_to_cli_args(d)
|
args = _flatten_to_cli_args(d)
|
||||||
assert "--optimizer.use_warmup=true" in args
|
assert "--optimizer.use_warmup=true" in args
|
||||||
assert "--optimizer.lr=0.01" in args
|
assert "--optimizer.lr=0.01" in args
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_removes_field_with_siblings_and_no_type():
|
||||||
|
"""Regression: when policy.path has siblings but no type:, the entire policy
|
||||||
|
block must still be removed from the cleaned config. Otherwise draccus tries
|
||||||
|
to decode the leftover dict as PreTrainedConfig and crashes on the missing
|
||||||
|
type discriminator.
|
||||||
|
"""
|
||||||
|
config = {
|
||||||
|
"dataset": {"repo_id": "lerobot/pusht"},
|
||||||
|
"policy": {
|
||||||
|
"path": "lerobot/smolvla_base",
|
||||||
|
"n_action_steps": 10,
|
||||||
|
"dtype": "bfloat16",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
|
||||||
|
yaml.dump(config, f)
|
||||||
|
config_path = f.name
|
||||||
|
|
||||||
|
_config_path_args.clear()
|
||||||
|
_config_yaml_overrides.clear()
|
||||||
|
cleaned_path = extract_path_fields_from_config(config_path, ["policy"])
|
||||||
|
|
||||||
|
with open(cleaned_path) as f:
|
||||||
|
cleaned = yaml.safe_load(f) or {}
|
||||||
|
assert "policy" not in cleaned, "policy block should be fully removed when path is present"
|
||||||
|
assert cleaned["dataset"]["repo_id"] == "lerobot/pusht"
|
||||||
|
assert _config_path_args["policy"] == "lerobot/smolvla_base"
|
||||||
|
overrides = get_yaml_overrides("policy")
|
||||||
|
assert any("n_action_steps=10" in o for o in overrides)
|
||||||
|
assert any("dtype=bfloat16" in o for o in overrides)
|
||||||
|
|
||||||
|
_config_path_args.clear()
|
||||||
|
_config_yaml_overrides.clear()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _DummyNested:
|
||||||
|
foo: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _DummyConfig:
|
||||||
|
nested: _DummyNested = field(default_factory=_DummyNested)
|
||||||
|
other: str = "default"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def __get_path_fields__(cls):
|
||||||
|
return ["nested"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_wrap_uses_cleaned_config_for_draccus_parse():
|
||||||
|
"""Regression: wrap() updates config_path_cli to point at the cleaned temp
|
||||||
|
file but must propagate that to the draccus.parse fallback branch. Without
|
||||||
|
the fix, cli_args still contains --config_path=<original> and draccus reads
|
||||||
|
the original YAML with `path:` still in it, crashing on the unknown field.
|
||||||
|
"""
|
||||||
|
config = {
|
||||||
|
"nested": {"path": "some/checkpoint", "foo": 42},
|
||||||
|
"other": "set-via-yaml",
|
||||||
|
}
|
||||||
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
|
||||||
|
yaml.dump(config, f)
|
||||||
|
config_path = f.name
|
||||||
|
|
||||||
|
_config_path_args.clear()
|
||||||
|
_config_yaml_overrides.clear()
|
||||||
|
|
||||||
|
captured: dict = {}
|
||||||
|
|
||||||
|
@parser.wrap()
|
||||||
|
def main(cfg: _DummyConfig) -> _DummyConfig:
|
||||||
|
captured["cfg"] = cfg
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
with patch.object(sys, "argv", ["prog", f"--config_path={config_path}"]):
|
||||||
|
main()
|
||||||
|
|
||||||
|
assert captured["cfg"].other == "set-via-yaml"
|
||||||
|
assert _config_path_args["nested"] == "some/checkpoint"
|
||||||
|
# Cleaned config dropped `nested:` entirely; defaults stand for this wrapper
|
||||||
|
# class (a real PreTrainedConfig would now load the checkpoint and apply
|
||||||
|
# the captured yaml_overrides via from_pretrained()).
|
||||||
|
assert captured["cfg"].nested.foo == 0
|
||||||
|
|
||||||
|
_config_path_args.clear()
|
||||||
|
_config_yaml_overrides.clear()
|
||||||
|
|||||||
Reference in New Issue
Block a user