remove add_special_tokens, not needed

This commit is contained in:
Pepijn
2025-09-19 14:57:45 +02:00
parent 2260e5a5c9
commit 8951e6034f

View File

@@ -1163,7 +1163,6 @@ class PI05OpenPIPolicy(PreTrainedPolicy):
full_prompts.append(full_prompt)
# Tokenize the full prompts with state
# Use the HuggingFace tokenizer properly (not .encode() which doesn't exist on AutoTokenizer)
tokenized = self.tokenizer(
full_prompts,
padding="max_length",
@@ -1171,7 +1170,6 @@ class PI05OpenPIPolicy(PreTrainedPolicy):
truncation=True,
max_length=self.max_token_len,
return_tensors="pt",
add_special_tokens=True,
)
tokens = tokenized["input_ids"].to(device)