mirror of
https://github.com/huggingface/lerobot.git
synced 2026-06-02 03:41:25 +00:00
beb22afd81
* **#2 — dedupe `_PLACEHOLDER_RE`.** The same regex was compiled in
`recipe.py` and `language_render.py`. Promote to module-level
`PLACEHOLDER_RE` in `recipe.py` (its primary owner — declares
template syntax) and import from `language_render.py`.
* **#3 — centralize language column names.** `io_utils.py` had
hardcoded `{"language_persistent", "language_events"}` literals at
two sites. Replace with `LANGUAGE_COLUMNS` import so a future column
rename can't silently desync.
* **#4 — defensive collate preserved-keys.** `lerobot_collate_fn`
silently filtered language fields from samples that didn't have
them, which would hand downstream consumers a preserved list
shorter than the tensor batch. Now: if any sample carries a key,
every sample in the batch must carry it; otherwise raise a
`ValueError` so the upstream rendering bug surfaces at the boundary.
* **#5 — `_scalar` rejects non-singleton lists.** Previously a zero-
or multi-element list fell through and triggered confusing
`float([])` errors downstream. Now raises `ValueError` with the
actual length.
* **#6 — refactor `_extract_complementary_data`.** Replace 11 lines
of `key = {... if ... else {}}` plus an 11-line splat dict with a
single `_COMPLEMENTARY_KEYS` tuple iterated once.
* **#7 — document `EXTENDED_STYLES`.** Was an empty `set()` with no
comment. Add a docstring explaining it's an intentional extension
point: downstream modules append project-local styles before
`column_for_style` is called.
* **#9 — `tools.mdx` notes the runtime layer is future work.** The
page referenced `src/lerobot/tools/`, `registry.py`, and
`get_tools(meta)` — none exist in this PR. Added a callout at the
start of "How to add your own tool" plus a note on the
implementations paragraph.
* **#10 — tests for YAML round-trip, malformed rows, blend
validation.** `test_recipe.py` grew from 1 case to 12 covering:
blend-or-messages exclusivity, target-turn requirement, blend
emptiness, weight presence/positivity, nested-blend rejection,
`from_dict` with nested blends, `from_yaml` / `load_recipe`
agreement, top-level non-mapping rejection. Added a malformed-row
test for `_normalize_rows` that asserts non-dict entries raise
`TypeError`.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
66 lines
2.4 KiB
Python
66 lines
2.4 KiB
Python
#!/usr/bin/env python
|
|
|
|
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from torch.utils.data._utils.collate import default_collate
|
|
|
|
from lerobot.datasets.language import LANGUAGE_COLUMNS
|
|
|
|
_PYTHON_LIST_KEYS = {"messages", "message_streams", "target_message_indices"}
|
|
|
|
|
|
def lerobot_collate_fn(batch: list[dict[str, Any] | None]) -> dict[str, Any] | None:
|
|
"""Collate function that preserves Python-list and language fields as lists.
|
|
|
|
Drops ``None`` samples (e.g. recipes that yielded no target message), keeps
|
|
rendered-message and language fields as plain Python lists, and delegates
|
|
every other key to PyTorch's ``default_collate``.
|
|
"""
|
|
batch = [sample for sample in batch if sample is not None]
|
|
if not batch:
|
|
return None
|
|
|
|
# All-or-nothing per key: a partial-presence batch (e.g. half the samples
|
|
# carry `messages` and half don't) is a real bug in the upstream
|
|
# rendering step — silently filtering would hand downstream consumers a
|
|
# preserved list shorter than the tensor batch. Raise instead so the
|
|
# mismatch surfaces at the boundary.
|
|
preserved: dict[str, list[Any]] = {}
|
|
for key in _PYTHON_LIST_KEYS:
|
|
presence = [key in sample for sample in batch]
|
|
if not any(presence):
|
|
continue
|
|
if not all(presence):
|
|
raise ValueError(
|
|
f"Inconsistent batch: {sum(presence)}/{len(batch)} samples carry {key!r}; "
|
|
f"every sample in a batch must agree."
|
|
)
|
|
preserved[key] = [sample[key] for sample in batch]
|
|
tensorizable = [
|
|
{
|
|
key: value
|
|
for key, value in sample.items()
|
|
if key not in _PYTHON_LIST_KEYS and key not in LANGUAGE_COLUMNS
|
|
}
|
|
for sample in batch
|
|
]
|
|
collated = default_collate(tensorizable)
|
|
collated.update(preserved)
|
|
return collated
|