ydy0615/lerobot-clone
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-06-01 19:31:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(processor): rename specialized processor -> XYZProcessorStep (#1852)

Browse Source
This commit is contained in:
Steven Palma
2025-09-03 17:30:47 +02:00
committed by GitHub
parent 8c796b39f5
commit d893bf1e30
20 changed files with 89 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/lerobot/processor/tokenizer_processor.py
View File

@@ -14,7 +14,7 @@ from lerobot.constants import OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS
from lerobot.utils.import_utils import _transformers_available
from .core import EnvTransition, TransitionKey
from .pipeline import ObservationProcessor, ProcessorStepRegistry
from .pipeline import ObservationProcessorStep, ProcessorStepRegistry
if TYPE_CHECKING or _transformers_available:
from transformers import AutoTokenizer
@@ -24,7 +24,7 @@ else:
@dataclass
@ProcessorStepRegistry.register(name="tokenizer_processor")
class TokenizerProcessor(ObservationProcessor):
class TokenizerProcessor(ObservationProcessorStep):
"""Tokenizes text tasks in complementary data using a huggingface tokenizer.
This processor handles tokenization of task strings found in the complementary_data