mirror of
https://github.com/huggingface/lerobot.git
synced 2026-06-03 12:21:27 +00:00
518e191337
Long episodes no longer get sparse subtasks. Previously a long episode
was subsampled to max_video_frames=32 across its whole duration (~1
frame/4s for a 2-min clip). New opt-in windowing keeps a CONSTANT
frames_per_second density by splitting the episode into fixed-length
windows and running the subtask chain per window.
New PlanConfig.subtask_window_seconds (default 0.0 = off). When > 0 and
the episode is longer than one window:
* episode is split into consecutive [w0, w1] windows of this length
* each window's frames are sampled at frames_per_second (so a 32s
window at 1 fps = 32 frames, filling but not exceeding the per-call
context budget)
* the full describe -> segment -> verify chain runs PER window, in
window-relative time [0, L]; spans are offset back to absolute
* all windows' spans are merged, frame-snap-deduped, and stitched into
one contiguous whole-episode cover
Implementation:
* _episode_video_block / _video_message / _describe_episode /
_verify_subtasks gain an optional window=(w0,w1); when set they
embed frames sampled in that absolute range at frames_per_second
(video_url path skipped — it's whole-episode).
* _clean_spans gains bounds= (override clamp range, for window-relative
spans) and dedupe= (skip frame-snap until the merged absolute set).
* new _generate_subtasks_windowed + _subtasks_for_window orchestrate
the loop; _generate_subtasks branches to them when window_s > 0.
run_hf_job.py: --plan.subtask_window_seconds=32 (32s windows at 1 fps).
Cost scales with episode length (chain calls × ceil(duration/window)).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
123 lines
5.6 KiB
Python
123 lines
5.6 KiB
Python
#!/usr/bin/env python
|
|
"""Launch ``lerobot-annotate`` on a Hugging Face job (vllm + Qwen3.6-27B VLM).
|
|
|
|
Spawns one ``h200x2`` job that:
|
|
|
|
1. installs this branch of ``lerobot`` plus the annotation extras,
|
|
2. boots two vllm servers (one per GPU) with Qwen3.6-27B (dense VLM),
|
|
3. runs the plan / interjections / vqa modules across the dataset
|
|
in free-form mode (each episode generates its own subtasks +
|
|
memory),
|
|
4. uploads the annotated dataset to ``--dest_repo_id`` (when set)
|
|
or back to ``--repo_id``.
|
|
|
|
Usage:
|
|
|
|
HF_TOKEN=hf_... uv run python examples/annotations/run_hf_job.py
|
|
|
|
Adjust ``CMD`` below to point at your own dataset / target hub repo.
|
|
"""
|
|
|
|
import os
|
|
|
|
from huggingface_hub import get_token, run_job
|
|
|
|
token = os.environ.get("HF_TOKEN") or get_token()
|
|
if not token:
|
|
raise RuntimeError("No HF token. Run `huggingface-cli login` or `export HF_TOKEN=hf_...`")
|
|
|
|
CMD = (
|
|
"apt-get update -qq && apt-get install -y -qq git ffmpeg && "
|
|
"pip install --no-deps "
|
|
"'lerobot @ git+https://github.com/huggingface/lerobot.git@feat/language-annotation-pipeline' && "
|
|
"pip install --upgrade-strategy only-if-needed "
|
|
"datasets pyarrow av jsonlines draccus gymnasium torchcodec mergedeep pyyaml-include toml typing-inspect "
|
|
"openai && "
|
|
"export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 && "
|
|
"export VLLM_VIDEO_BACKEND=pyav && "
|
|
"lerobot-annotate "
|
|
"--repo_id=pepijn223/robocasa_smoke_2atomic_v3 "
|
|
"--dest_repo_id=pepijn223/robocasa_smoke_2atomic_v3_ann "
|
|
"--push_to_hub=true "
|
|
"--vlm.backend=openai "
|
|
"--vlm.model_id=Qwen/Qwen3.6-27B "
|
|
"--vlm.parallel_servers=2 "
|
|
"--vlm.num_gpus=2 "
|
|
'--vlm.serve_command="vllm serve Qwen/Qwen3.6-27B '
|
|
"--tensor-parallel-size 1 --max-model-len 32768 "
|
|
'--gpu-memory-utilization 0.8 --uvicorn-log-level warning --port {port}" '
|
|
"--vlm.serve_ready_timeout_s=1800 "
|
|
"--vlm.client_concurrency=128 "
|
|
"--vlm.max_new_tokens=512 "
|
|
"--vlm.temperature=0.7 "
|
|
"--executor.episode_parallelism=16 "
|
|
"--vlm.chat_template_kwargs='{\"enable_thinking\": false}' "
|
|
"--vlm.camera_key=observation.images.robot0_agentview_right "
|
|
# Phase 1 — plan module (subtasks + plan + memory).
|
|
# Embed decoded frames directly (use_video_url=false) rather than
|
|
# handing the server a file:// clip. The embedded path is more
|
|
# reliable: if clip extraction ever fails, the video_url path would
|
|
# silently send NO video and the VLM would hallucinate subtasks from
|
|
# the task text alone.
|
|
#
|
|
# CONTEXT BUDGET: with embedded frames, each frame is ~250-320 vision
|
|
# tokens. The model's context is 32768 (see --max-model-len). 32
|
|
# frames sampled uniformly across the episode (~8-10k tokens) fits
|
|
# comfortably alongside the prompt and the describe/verify passes.
|
|
# Do NOT raise max_video_frames toward 128 with embedded frames — that
|
|
# is ~33-39k tokens and overflows the context (BadRequestError 400,
|
|
# "Input length exceeds maximum context length").
|
|
"--plan.use_video_url=false "
|
|
"--plan.frames_per_second=1.0 "
|
|
"--plan.max_video_frames=32 "
|
|
# Constant 1 fps density via windowing: episodes longer than 32s are
|
|
# split into 32-second windows (each 32 frames @ 1 fps, fits context),
|
|
# so long episodes get MORE subtasks instead of a sparser whole-episode
|
|
# view. describe->segment->verify runs per window; spans are merged +
|
|
# stitched to a contiguous whole-episode cover. 0 disables.
|
|
"--plan.subtask_window_seconds=32 "
|
|
# IMPORTANT for RoboCasa: the dataset's task string ("Navigate to the
|
|
# stove", "Pick the mug...") is authoritative and is what eval uses.
|
|
# ``derive_task_from_video=off`` keeps that canonical task driving
|
|
# subtask generation. Do NOT use ``always`` here — it throws the real
|
|
# task away, asks the VLM "what is this video about?" with no hint,
|
|
# and the hallucinated task then poisons every subtask + plan row.
|
|
"--plan.derive_task_from_video=off "
|
|
# NO task augmentation for RoboCasa: eval conditions on the exact task
|
|
# strings, so synthetic rephrasings are unused at best and (when they
|
|
# drift, e.g. "wander around the kitchen") harmful. 0 rephrasings +
|
|
# axes disabled = the policy only ever sees the canonical task.
|
|
"--plan.n_task_rephrasings=0 "
|
|
# action_records OFF: the structured {verb,object,arm,grasp,dest}
|
|
# schema is a manipulation schema; RoboCasa navigation / atomic tasks
|
|
# don't fit it and the VLM hallucinates. When on, records are purely
|
|
# additive (emitted as style="action_record" rows) and never touch
|
|
# the subtask text — useful only for long composite manipulation
|
|
# tasks. Leave off for RoboCasa atomic / navigation.
|
|
# Keep subtask decomposition tight for atomic tasks:
|
|
"--plan.plan_max_steps=6 "
|
|
# NOTE: the multi-call subtask quality chain (describe -> segment ->
|
|
# verify, 3 VLM calls/episode) is ON BY DEFAULT now. Pass
|
|
# --plan.subtask_describe_first=false / --plan.subtask_verify=false to
|
|
# disable on datasets you've verified are easy and want fewer calls.
|
|
# Phase 2 — interjections + speech.
|
|
"--interjections.max_interjections_per_episode=6 "
|
|
# Phase 4 — general VQA.
|
|
# Ground VQA on the SAME single camera as plan/interjections
|
|
# (--vlm.camera_key) instead of iterating every camera. The whole
|
|
# pipeline then focuses on one view, e.g. observation.images.base.
|
|
"--vqa.restrict_to_default_camera=true "
|
|
"--vqa.K=1 "
|
|
"--vqa.vqa_emission_hz=1.0"
|
|
)
|
|
|
|
job = run_job(
|
|
image="vllm/vllm-openai:latest",
|
|
command=["bash", "-c", CMD],
|
|
flavor="h200x2",
|
|
secrets={"HF_TOKEN": token},
|
|
timeout="2h",
|
|
)
|
|
print(f"Job URL: {job.url}")
|
|
print(f"Job ID: {job.id}")
|