mirror of
https://github.com/huggingface/lerobot.git
synced 2026-05-31 02:41:24 +00:00
add: number of workers for merging datasets
This commit is contained in:
@@ -103,6 +103,7 @@ class SplitConfig:
|
||||
class MergeConfig:
|
||||
type: str = "merge"
|
||||
repo_ids: list[str] | None = None
|
||||
num_workers: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -215,6 +216,7 @@ def handle_merge(cfg: EditDatasetConfig) -> None:
|
||||
datasets,
|
||||
output_repo_id=cfg.repo_id,
|
||||
output_dir=output_dir,
|
||||
num_workers=cfg.operation.num_workers,
|
||||
)
|
||||
|
||||
logging.info(f"Merged dataset saved to {output_dir}")
|
||||
|
||||
Reference in New Issue
Block a user