2026-04-08 17:48:58 +02:00
|
|
|
"""Tests for the benchmark dispatch refactor (create_envs / get_env_processors on EnvConfig)."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import logging
|
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
|
|
|
|
|
|
import gymnasium as gym
|
|
|
|
|
import pytest
|
|
|
|
|
from gymnasium.envs.registration import register, registry as gym_registry
|
|
|
|
|
|
|
|
|
|
from lerobot.configs.types import PolicyFeature
|
|
|
|
|
from lerobot.envs.configs import EnvConfig
|
|
|
|
|
from lerobot.envs.factory import make_env, make_env_config, make_env_pre_post_processors
|
|
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_registry_all_types():
|
|
|
|
|
"""make_env_config should resolve every registered EnvConfig subclass via the registry."""
|
|
|
|
|
known = list(EnvConfig.get_known_choices().keys())
|
|
|
|
|
assert len(known) >= 6
|
|
|
|
|
for t in known:
|
|
|
|
|
cfg = make_env_config(t)
|
|
|
|
|
if not isinstance(cfg, EnvConfig):
|
|
|
|
|
continue
|
|
|
|
|
assert cfg.type == t
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_unknown_type():
|
|
|
|
|
with pytest.raises(ValueError, match="not registered"):
|
|
|
|
|
make_env_config("nonexistent")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_identity_processors():
|
|
|
|
|
"""Base class get_env_processors() returns identity pipelines."""
|
|
|
|
|
cfg = make_env_config("aloha")
|
|
|
|
|
pre, post = cfg.get_env_processors()
|
|
|
|
|
assert len(pre.steps) == 0 and len(post.steps) == 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_delegation():
|
|
|
|
|
"""make_env() should call cfg.create_envs(), not use if/elif dispatch."""
|
|
|
|
|
sentinel = {"delegated": {0: "marker"}}
|
|
|
|
|
fake = type(
|
|
|
|
|
"Fake",
|
|
|
|
|
(),
|
|
|
|
|
{
|
|
|
|
|
"hub_path": None,
|
|
|
|
|
"create_envs": lambda self, n_envs, use_async_envs=False: sentinel,
|
|
|
|
|
},
|
|
|
|
|
)()
|
|
|
|
|
result = make_env(fake, n_envs=1)
|
|
|
|
|
assert result is sentinel
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_processors_delegation():
|
|
|
|
|
"""make_env_pre_post_processors delegates to cfg.get_env_processors()."""
|
|
|
|
|
cfg = make_env_config("aloha")
|
|
|
|
|
pre, post = make_env_pre_post_processors(cfg, policy_cfg=None)
|
|
|
|
|
assert len(pre.steps) == 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_base_create_envs():
|
|
|
|
|
"""Base class create_envs() should build a single-task VectorEnv via gym.make()."""
|
|
|
|
|
gym_id = "_dispatch_test/CartPole-v99"
|
|
|
|
|
if gym_id not in gym_registry:
|
|
|
|
|
register(id=gym_id, entry_point="gymnasium.envs.classic_control:CartPoleEnv")
|
|
|
|
|
|
|
|
|
|
@EnvConfig.register_subclass("_dispatch_base_test")
|
|
|
|
|
@dataclass
|
|
|
|
|
class _Env(EnvConfig):
|
|
|
|
|
task: str = "CartPole-v99"
|
|
|
|
|
fps: int = 10
|
|
|
|
|
features: dict[str, PolicyFeature] = field(default_factory=dict)
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def package_name(self):
|
|
|
|
|
return "_dispatch_test"
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def gym_id(self):
|
|
|
|
|
return gym_id
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def gym_kwargs(self):
|
|
|
|
|
return {}
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
envs = _Env().create_envs(n_envs=2)
|
|
|
|
|
assert "_dispatch_base_test" in envs
|
|
|
|
|
env = envs["_dispatch_base_test"][0]
|
feat(envs): lazy env init + AsyncVectorEnv as default for n_envs > 1 (#3274)
* docs(benchmarks): add benchmark integration guide and standardize benchmark docs
Add a comprehensive guide for adding new benchmarks to LeRobot, and
refactor the existing LIBERO and Meta-World docs to follow the new
standardized template.
Made-with: Cursor
* refactor(envs): move dispatch logic from factory into EnvConfig subclasses
Replace hardcoded if/elif chains in factory.py with create_envs() and
get_env_processors() methods on EnvConfig. New benchmarks now only need
to register a config subclass — no factory.py edits required.
Net -23 lines: factory.py shrinks from ~200 to ~70 lines of logic.
Made-with: Cursor
* docs(benchmarks): clean up adding-benchmarks guide for clarity
Rewrite for simpler language, better structure, and easier navigation.
Move quick-reference table to the top, fold eval explanation into
architecture section, condense the doc template to a bulleted outline.
Made-with: Cursor
* fix link
* fix task count
* fix: enable SmolVLA eval on LIBERO with custom camera mappings
- Thread camera_name_mapping from LiberoEnv config through to gym envs
- Sync features_map with camera_name_mapping in LiberoEnv.__post_init__
- Fix render() to use first available camera instead of hardcoded "image"
- Handle non-dict final_info in rollout by falling back to info["is_success"]
- Add use_peft legacy field to SmolVLAConfig for checkpoint compat
- Add defaults to GR00TN15Config init=False fields for transformers 5.3
Made-with: Cursor
* fix: use direct AutoresetMode import for gymnasium compat
Made-with: Cursor
* fix: handle gymnasium < 1.0 without AutoresetMode
Made-with: Cursor
* refactor: revert policy changes, keep env-only camera mapping fixes
- Revert GR00T N1.5 default_factory/default changes (transformers compat)
- Revert SmolVLA use_peft legacy field
- Apply ruff formatting fixes
- camera_name_mapping stays entirely in env/eval layer (no policy changes)
Made-with: Cursor
* Update docs/source/env_processor.mdx
Co-authored-by: Khalil Meftah <khalil.meftah@huggingface.co>
Signed-off-by: Pepijn <138571049+pkooij@users.noreply.github.com>
* feat(envs): lazy env init + AsyncVectorEnv as default for n_envs > 1
LiberoEnv and MetaworldEnv previously allocated GPU resources (EGL context,
OpenGL framebuffer) in __init__, before AsyncVectorEnv's fork(). Worker
processes inherited stale GPU handles, causing EGL_BAD_CONTEXT crashes on
first render.
Fix: defer OffScreenRenderEnv / MT1 construction to _ensure_env(), called on
first reset() or step() inside the worker subprocess. Each worker creates its
own clean context after fork().
Also fixes lerobot_eval.py:170 (add_envs_task TODO): replace with
env.call("task") which works with both SyncVectorEnv and AsyncVectorEnv.
AsyncVectorEnv is now the default for n_envs > 1; auto-downgraded to
SyncVectorEnv when n_envs=1 (no benefit, less overhead).
Expected speedup: ~15-20x for LIBERO Spatial with batch_size=50.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix: close envs between tasks to prevent worker process accumulation
eval_policy_all never closed environments after each task completed,
causing AsyncVectorEnv worker processes to accumulate (N_tasks × n_envs).
This led to OOM, BrokenPipeError and EOFError on multi-task benchmarks.
Also fixes:
- AsyncVectorEnv compat in envs/utils.py (use get_attr/call instead of .envs)
- Tuple task handling in tokenizer_processor and lerobot_eval
- _LazyAsyncVectorEnv for deferred worker spawning in LIBERO
Made-with: Cursor
* fix(eval): use task_description instead of task for language conditioning
env.call("task") returns the LIBERO task name with underscores
(e.g. "pick_up_the_black_bowl_...") instead of the natural language
description ("pick up the black bowl ..."). The VLM tokenizes these
completely differently, causing 0.0 reward across all episodes.
Made-with: Cursor
* docs: update adding_benchmarks for async env changes
- Replace add_envs_task reference with env.call("task_description")
- Update use_async_envs default to True
- Add note about lazy GPU init for AsyncVectorEnv compatibility
Made-with: Cursor
* feat(eval): batch_size=auto + faster env loading
- batch_size=0 (default) auto-tunes based on CPU cores, capped by
n_episodes and 64. Removes the need for users to guess the right
value. The old batch_size > n_episodes error is replaced by silently
clamping to n_episodes.
- _LazyAsyncVectorEnv accepts pre-computed spaces so only one temp env
is created per suite (not per task). For libero_spatial (10 tasks)
this avoids 9 redundant LiberoEnv instantiations during env setup.
Made-with: Cursor
* docs: add evaluation guide and update benchmarks doc
- New docs/source/evaluation.mdx covering lerobot-eval usage, batch_size
auto-tuning, AsyncVectorEnv performance, tuning tips, output format,
multi-task evaluation, and programmatic usage.
- Add evaluation page to _toctree.yml under Benchmarks section.
- Update adding_benchmarks.mdx to reference batch_size auto default and
link to the evaluation guide.
Made-with: Cursor
* docs(evaluation): remove benchmark table, rename section header
Made-with: Cursor
* perf(eval): shared memory, observation passthrough, task prefetch
- AsyncVectorEnv now uses shared_memory=True for zero-copy observation transfer
- LiberoEnvConfig.gym_kwargs passes observation_height/width to the env
- eval_policy_all prefetches next task's workers while current task runs
Made-with: Cursor
* style: ruff format
Made-with: Cursor
* chore: revert env_processor.mdx changes (not part of this PR)
Made-with: Cursor
* ci(benchmarks): add isolated integration tests for libero and metaworld
Each benchmark gets its own Docker image (lerobot[libero] / lerobot[metaworld]
only) so incompatible dep trees cannot collide. A 1-episode smoke eval runs
per benchmark on GPU runners.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* ci(benchmarks): pin action hashes and use uv sync --locked
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* ci(benchmarks): trigger only on envs/ or lerobot_eval.py changes
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix(ci): set LIBERO_DATA_FOLDER to bypass interactive stdin prompt
libero/__init__.py calls input() to ask about a custom dataset path,
which raises EOFError when stdin is closed inside Docker. Setting
LIBERO_DATA_FOLDER skips the prompt entirely.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* docs(benchmarks): add CI smoke test step to adding_benchmarks guide
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix(ci): pre-create libero config in Dockerfile to bypass stdin prompt
libero/__init__.py calls input() when ~/.libero/config.yaml is missing.
We write the config at image build time (without importing libero) so
the prompt never fires at runtime. Also trigger CI on pyproject.toml changes.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix(ci): use shell to create libero config instead of multiline python -c
The multiline RUN python -c "..." was being parsed as Dockerfile
instructions. Use printf to write ~/.libero/config.yaml directly.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix(ci): point libero config to bundled package init_files
The config was pointing to /tmp/libero_init which doesn't exist.
Use importlib.util.find_spec to locate the hf-libero package directory
and write paths to the actual bundled bddl_files/init_files/assets.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix(ci): add smolvla extra to benchmark Dockerfiles
num2words (required by SmolVLM processor) is declared in lerobot[smolvla],
not lerobot[libero/metaworld]. Install both extras together.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix(eval): render_frame covers _LazyAsyncVectorEnv
isinstance(env, AsyncVectorEnv) silently skipped _LazyAsyncVectorEnv,
causing video rendering to produce no frames on the default async path.
Switch to hasattr(env, "call") so any async-compatible env (including
_LazyAsyncVectorEnv) hits the call("render") branch.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* refactor(envs): remove unused _get_sub_env_attr helper
_get_sub_env_attr was defined but never called anywhere in the codebase.
_sub_env_has_attr (its sibling) is kept — it is actively used in utils.py.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* chore: apply prettier formatting to docs
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* docs(env_processor): remove deprecated add_envs_task from pipeline example
add_envs_task is replaced by env.call("task_description") in this PR.
Remove it from the pipeline walkthrough and renumber the steps (8→7).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* refactor(envs): remove __del__ from _LazyAsyncVectorEnv
__del__ is unreliable as a cleanup mechanism. close() is already called
explicitly in the eval loop's finally block, so the finalizer is redundant.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix(eval): prefetch next task's workers after close to avoid GPU memory overlap
Previously, next task's AsyncVectorEnv workers were spawned while the
current task was still running, causing both tasks' GPU contexts to coexist.
Moving the prefetch start into the finally block (after env.close()) ensures
workers for task N+1 only spin up once task N has released GPU memory.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* refactor(envs): move _LazyAsyncVectorEnv to utils and apply to metaworld
_LazyAsyncVectorEnv lived in libero.py but metaworld had the same OOM
problem: all tasks' AsyncVectorEnv workers were spawned eagerly, wasting
GPU memory for tasks not yet running.
Move the class to envs/utils.py so both environments share it, then apply
the same is_async + lazy wrapping pattern in create_metaworld_envs.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* chore: remove out-of-scope benchmark/CI/docs files from PR
Benchmark CI workflow, Dockerfiles, benchmark docs, evaluation smoke-test
doc, and dispatch tests belong in a separate PR. Scope this PR to the
async env init changes only.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* chore: restore adding_benchmarks + test_dispatch, drop env_processor changes
- Restore docs/source/adding_benchmarks.mdx (belongs in this PR)
- Restore tests/envs/test_dispatch.py (belongs in this PR)
- Revert docs/source/env_processor.mdx to main (out of scope for this PR)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* docs(adding_benchmarks): remove CI smoke test step (coming in separate PR)
Step 7 (Dockerfile + benchmark_tests.yml CI job) and its table rows are
out of scope for this PR. The CI infrastructure will be added on top in a
follow-up PR.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* refactor(envs): remove unused add_envs_task
Replaced by env.call("task_description") in lerobot_eval.py. No callers
remain in the codebase.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* style: fix prettier formatting in env_processor.mdx
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix(eval): catch AttributeError and NotImplementedError explicitly for task description
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix(envs): use forkserver context and close envs in test to prevent deadlock
AsyncVectorEnv with default fork context leaks worker processes between
test_policy parametrized cases; subsequent env creation deadlocks because
new forked workers inherit stale pipe FDs from previous test's leaked workers.
- configs.py: pass context="forkserver" to AsyncVectorEnv (matches _LazyAsyncVectorEnv)
- test_policies.py: call close_envs(envs) at end of test_policy to clean up workers
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
* fix(envs): default use_async_envs=False in create_envs and make_env
Tests that call make_env(n_envs=2) without passing use_async_envs were
getting AsyncVectorEnv, whose forked workers can't resolve gym namespaces
registered at runtime. Default to False (sync) so existing tests pass.
lerobot_eval.py explicitly passes cfg.eval.use_async_envs, so the CLI
async behaviour (controlled by EvalConfig.use_async_envs) is unchanged.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---------
Signed-off-by: Pepijn <138571049+pkooij@users.noreply.github.com>
Co-authored-by: Khalil Meftah <khalil.meftah@huggingface.co>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-09 10:29:20 +02:00
|
|
|
assert isinstance(env, gym.vector.VectorEnv)
|
2026-04-08 17:48:58 +02:00
|
|
|
assert env.num_envs == 2
|
|
|
|
|
env.close()
|
|
|
|
|
finally:
|
|
|
|
|
if gym_id in gym_registry:
|
|
|
|
|
del gym_registry[gym_id]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_custom_create_envs_override():
|
|
|
|
|
"""A custom EnvConfig subclass can override create_envs()."""
|
|
|
|
|
mock_vec = gym.vector.SyncVectorEnv([lambda: gym.make("CartPole-v1")])
|
|
|
|
|
|
|
|
|
|
@EnvConfig.register_subclass("_dispatch_custom_test")
|
|
|
|
|
@dataclass
|
|
|
|
|
class _Env(EnvConfig):
|
|
|
|
|
task: str = "x"
|
|
|
|
|
features: dict[str, PolicyFeature] = field(default_factory=dict)
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def gym_kwargs(self):
|
|
|
|
|
return {}
|
|
|
|
|
|
|
|
|
|
def create_envs(self, n_envs, use_async_envs=False):
|
|
|
|
|
return {"custom_suite": {0: mock_vec}}
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
result = make_env(_Env(), n_envs=1)
|
|
|
|
|
assert "custom_suite" in result
|
|
|
|
|
finally:
|
|
|
|
|
mock_vec.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_custom_get_env_processors_override():
|
|
|
|
|
"""A custom EnvConfig subclass can override get_env_processors()."""
|
|
|
|
|
from lerobot.processor.pipeline import DataProcessorPipeline
|
|
|
|
|
|
|
|
|
|
@EnvConfig.register_subclass("_dispatch_proc_test")
|
|
|
|
|
@dataclass
|
|
|
|
|
class _Env(EnvConfig):
|
|
|
|
|
task: str = "x"
|
|
|
|
|
features: dict[str, PolicyFeature] = field(default_factory=dict)
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def gym_kwargs(self):
|
|
|
|
|
return {}
|
|
|
|
|
|
|
|
|
|
def get_env_processors(self):
|
|
|
|
|
return DataProcessorPipeline(steps=[]), DataProcessorPipeline(steps=[])
|
|
|
|
|
|
|
|
|
|
pre, post = _Env().get_env_processors()
|
|
|
|
|
assert isinstance(pre, DataProcessorPipeline)
|