Compare commits

..

4 Commits

Author SHA1 Message Date
Jade Choghari
c11d8f1bb6 fix style 2025-11-11 00:17:31 +01:00
Jade Choghari
6001b2c3ad add libero testing 2025-11-11 00:13:54 +01:00
Steven Palma
a5b29d4301 chore(installation): remove libero installation patch (#2416)
* chore(installation): remove libero installation patch

* fix(ci): exclude groot for unbound deps test
2025-11-10 11:51:52 +01:00
Steven Palma
a4aa316470 fix(dataset): fix data access bottleneck for faster training (#2408) 2025-11-07 21:54:44 +01:00
9 changed files with 206 additions and 55 deletions

View File

@@ -83,11 +83,11 @@ jobs:
fi
- name: Remove Tags with Git dependencies
# TODO(Steven): Temporary patch to remove libero and pi from PyPi 0.4.0 release due to its reliance on git dependencies.
# TODO(Steven): Temporary patch to remove pi from PyPi 0.4.0 release due to its reliance on git dependencies.
run: |
echo "::info:: Checking for Git dependencies to remove from pyproject.toml..."
grep -E '@ git\+https|lerobot\[pi\]|lerobot\[libero\]' pyproject.toml | sed 's/^/::warning:: Removing line: /' || true
sed -E -i '/@ git\+https|lerobot\[pi\]|lerobot\[libero\]/d' pyproject.toml
grep -E '@ git\+https|lerobot\[pi\]' pyproject.toml | sed 's/^/::warning:: Removing line: /' || true
sed -E -i '/@ git\+https|lerobot\[pi\]/d' pyproject.toml
echo "::info:: Git dependencies removed. Proceeding with build."
- name: Install build dependencies

View File

@@ -70,7 +70,7 @@ jobs:
echo "Dependencies unbound:" && cat pyproject.toml
- name: Install lerobot with all extras
run: uv sync --all-extras
run: uv sync --all-extras --no-extra groot # TODO(Steven): Make flash-attn optional
- name: Run pytest (all extras)
run: uv run pytest tests -vv

View File

@@ -186,7 +186,7 @@ For a full list of optional dependencies, see:
https://pypi.org/project/lerobot/
> [!NOTE]
> For lerobot 0.4.0, if you want to install libero or pi tags, you will have to do: `pip install "lerobot[pi,libero]@git+https://github.com/huggingface/lerobot.git"`.
> For lerobot 0.4.0, if you want to install pi tags, you will have to do: `pip install "lerobot[pi]@git+https://github.com/huggingface/lerobot.git"`.
>
> This will be solved in the next patch release

View File

@@ -82,7 +82,7 @@ For a full list of optional dependencies, see:
https://pypi.org/project/lerobot/
> [!NOTE]
> For lerobot 0.4.0, if you want to install libero or pi, you will have to do: `pip install "lerobot[pi,libero]@git+https://github.com/huggingface/lerobot.git"`
> For lerobot 0.4.0, if you want to install pi, you will have to do: `pip install "lerobot[pi]@git+https://github.com/huggingface/lerobot.git"`
### Troubleshooting

View File

@@ -28,11 +28,6 @@ LIBERO is now part of our **multi-eval supported simulation**, meaning you can b
To Install LIBERO, after following LeRobot official instructions, just do:
`pip install -e ".[libero]"`
> [!NOTE]
> For lerobot 0.4.0, if you want to install libero tag, you will have to do: `pip install "lerobot[libero]@git+https://github.com/huggingface/lerobot.git"`.
>
> This will be solved in the next patch release
### Single-suite evaluation
Evaluate a policy on one LIBERO suite:

View File

@@ -23,8 +23,6 @@ import platform
import time
from pathlib import Path
from threading import Event, Lock, Thread
from multiprocessing import Process, Event as EventProcess, JoinableQueue as Queue
from queue import Empty
from typing import Any
from numpy.typing import NDArray # type: ignore # TODO: add type stubs for numpy.typing
@@ -121,10 +119,11 @@ class OpenCVCamera(Camera):
self.videocapture: cv2.VideoCapture | None = None
self.process: Process | None = None
self.stop_event: EventProcess | None = None
self.frame_queue: Queue = Queue()
self.thread: Thread | None = None
self.stop_event: Event | None = None
self.frame_lock: Lock = Lock()
self.latest_frame: NDArray[Any] | None = None
self.new_frame_event: Event = Event()
self.rotation: int | None = get_cv2_rotation(config.rotation)
self.backend: int = get_cv2_backend()
@@ -443,36 +442,37 @@ class OpenCVCamera(Camera):
while not self.stop_event.is_set():
try:
color_image = self.read()
self.frame_queue.put_nowait(color_image)
with self.frame_lock:
self.latest_frame = color_image
self.new_frame_event.set()
except DeviceNotConnectedError:
break
except Exception as e:
logger.warning(f"Error reading frame in background thread for {self}: {e}")
def _start_read_process(self) -> None:
def _start_read_thread(self) -> None:
"""Starts or restarts the background read thread if it's not running."""
if self.process is not None and self.process.is_alive():
self.frame_queue.join()
self.process.join()
if self.thread is not None and self.thread.is_alive():
self.thread.join(timeout=0.1)
if self.stop_event is not None:
self.stop_event.set()
self.stop_event = Event()
self.process = Process(target=self._read_loop, args=(), name=f"{self}_read_loop")
self.process.daemon = True
self.process.start()
self.thread = Thread(target=self._read_loop, args=(), name=f"{self}_read_loop")
self.thread.daemon = True
self.thread.start()
def _stop_read_thread(self) -> None:
"""Signals the background read thread to stop and waits for it to join."""
if self.stop_event is not None:
self.stop_event.set()
if self.process is not None and self.process.is_alive():
self.frame_queue.join()
self.process.join()
if self.thread is not None and self.thread.is_alive():
self.thread.join(timeout=2.0)
self.process = None
self.thread = None
self.stop_event = None
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
@@ -499,32 +499,24 @@ class OpenCVCamera(Camera):
if not self.is_connected:
raise DeviceNotConnectedError(f"{self} is not connected.")
if self.process is None or not self.process.is_alive():
self._start_read_process()
if self.thread is None or not self.thread.is_alive():
self._start_read_thread()
if self.latest_frame is None:
self.latest_frame = self.frame_queue.get()
self.frame_queue.task_done()
return self.latest_frame
if not self.new_frame_event.wait(timeout=timeout_ms / 1000.0):
thread_alive = self.thread is not None and self.thread.is_alive()
raise TimeoutError(
f"Timed out waiting for frame from camera {self} after {timeout_ms} ms. "
f"Read thread alive: {thread_alive}."
)
try:
frame = self.frame_queue.get(timeout=timeout_ms / 1000.0)
self.frame_queue.task_done()
except Empty:
process_alive = self.process is not None and self.process.is_alive()
if process_alive:
logger.warning(f"{self} async_read timed out after {timeout_ms} ms but camera is still running.")
return self.latest_frame
else:
raise TimeoutError(
f"{self} async_read timed out after {timeout_ms} ms: camera is not responding !"
)
with self.frame_lock:
frame = self.latest_frame
self.new_frame_event.clear()
if frame is None:
raise RuntimeError(f"Internal error: Event set but no frame available for {self}.")
else:
self.latest_frame = frame
return self.latest_frame
return frame
def disconnect(self) -> None:
"""

View File

@@ -940,11 +940,26 @@ class LeRobotDataset(torch.utils.data.Dataset):
return query_timestamps
def _query_hf_dataset(self, query_indices: dict[str, list[int]]) -> dict:
return {
key: torch.stack(self.hf_dataset[q_idx][key])
for key, q_idx in query_indices.items()
if key not in self.meta.video_keys
}
"""
Query dataset for indices across keys, skipping video keys.
Tries column-first [key][indices] for speed, falls back to row-first.
Args:
query_indices: Dict mapping keys to index lists to retrieve
Returns:
Dict with stacked tensors of queried data (video keys excluded)
"""
result: dict = {}
for key, q_idx in query_indices.items():
if key in self.meta.video_keys:
continue
try:
result[key] = torch.stack(self.hf_dataset[key][q_idx])
except (KeyError, TypeError, IndexError):
result[key] = torch.stack(self.hf_dataset[q_idx][key])
return result
def _query_videos(self, query_timestamps: dict[str, list[float]], ep_idx: int) -> dict[str, torch.Tensor]:
"""Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function

View File

@@ -237,9 +237,10 @@ class LiberoEnv(gym.Env):
def reset(self, seed=None, **kwargs):
super().reset(seed=seed)
self._env.seed(seed)
raw_obs = self._env.reset()
if self.init_states and self._init_states is not None:
self._env.set_init_state(self._init_states[self._init_state_id])
raw_obs = self._env.reset()
raw_obs = self._env.env._get_observations()
# After reset, objects may be unstable (slightly floating, intersecting, etc.).
# Step the simulator with a no-op action for a few frames so everything settles.

148
tests/envs/test_libero.py Normal file
View File

@@ -0,0 +1,148 @@
#!/usr/bin/env python
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import numpy as np
import pytest
from lerobot.envs.factory import make_env, make_env_config
# Set MuJoCo rendering backend before importing environment
os.environ["MUJOCO_GL"] = "egl"
def assert_observations_equal(obs1, obs2, path="", atol=1e-8):
"""
Recursively compare two observations and assert they are equal.
Args:
obs1: First observation (dict or numpy array)
obs2: Second observation (dict or numpy array)
path: Current path in nested structure (for error messages)
atol: Absolute tolerance for numpy array comparisons
"""
if isinstance(obs1, dict) and isinstance(obs2, dict):
assert obs1.keys() == obs2.keys(), f"Keys differ at {path}: {obs1.keys()} != {obs2.keys()}"
for key in obs1:
assert_observations_equal(obs1[key], obs2[key], path=f"{path}.{key}" if path else key, atol=atol)
elif isinstance(obs1, np.ndarray) and isinstance(obs2, np.ndarray):
assert obs1.shape == obs2.shape, f"Shape mismatch at {path}: {obs1.shape} != {obs2.shape}"
assert obs1.dtype == obs2.dtype, f"Dtype mismatch at {path}: {obs1.dtype} != {obs2.dtype}"
assert np.allclose(obs1, obs2, atol=atol), (
f"Array values differ at {path}: max abs diff = {np.abs(obs1 - obs2).max()}"
)
else:
assert type(obs1) is type(obs2), f"Type mismatch at {path}: {type(obs1)} != {type(obs2)}"
assert obs1 == obs2, f"Values differ at {path}: {obs1} != {obs2}"
def test_libero_env_creation():
"""Test that the libero environment can be created successfully."""
config = make_env_config("libero", task="libero_spatial")
envs_dict = make_env(config)
assert "libero_spatial" in envs_dict
assert 0 in envs_dict["libero_spatial"]
env = envs_dict["libero_spatial"][0]
assert env is not None
# Test basic reset
observation, info = env.reset(seed=42)
assert observation is not None
assert info is not None
env.close()
def test_libero_reset_determinism():
"""Test that resetting with the same seed produces identical observations."""
config = make_env_config("libero", task="libero_spatial")
envs_dict = make_env(config)
env = envs_dict["libero_spatial"][0]
# Reset multiple times with the same seed
obs1, info1 = env.reset(seed=42)
obs2, info2 = env.reset(seed=42)
obs3, info3 = env.reset(seed=42)
# All observations should be identical
assert_observations_equal(obs1, obs2)
assert_observations_equal(obs1, obs3)
assert_observations_equal(obs2, obs3)
env.close()
def test_libero_step_determinism():
"""Test that step() is deterministic when resetting with the same seed."""
config = make_env_config("libero", task="libero_spatial")
envs_dict = make_env(config)
env = envs_dict["libero_spatial"][0]
seed = 42
# First rollout
obs1, info1 = env.reset(seed=seed)
action = env.action_space.sample()
obs_after_step1, reward1, terminated1, truncated1, info_step1 = env.step(action)
# Second rollout with identical seed and action
obs2, info2 = env.reset(seed=seed)
obs_after_step2, reward2, terminated2, truncated2, info_step2 = env.step(action)
# Initial observations should be identical
assert_observations_equal(obs1, obs2)
# Post-step observations should be identical
assert_observations_equal(obs_after_step1, obs_after_step2)
# Rewards and termination flags should be identical
assert np.allclose(reward1, reward2), f"Rewards differ: {reward1} != {reward2}"
assert np.array_equal(terminated1, terminated2), (
f"Terminated flags differ: {terminated1} != {terminated2}"
)
assert np.array_equal(truncated1, truncated2), f"Truncated flags differ: {truncated1} != {truncated2}"
env.close()
@pytest.mark.parametrize("task", ["libero_spatial", "libero_object", "libero_goal", "libero_10"])
def test_libero_tasks(task):
"""Test that different libero tasks can be created and used."""
config = make_env_config("libero", task=task)
envs_dict = make_env(config)
assert task in envs_dict
assert 0 in envs_dict[task]
env = envs_dict[task][0]
observation, info = env.reset(seed=42)
assert observation is not None
assert info is not None
# Take a step
action = env.action_space.sample()
obs, reward, terminated, truncated, info = env.step(action)
assert obs is not None
assert reward is not None
assert isinstance(terminated, (bool, np.ndarray))
assert isinstance(truncated, (bool, np.ndarray))
env.close()