vast.ai runbook + per-tokenizer eos handling in loop_common

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-14 01:14:57 +02:00
parent ef9c08966c
commit 69ce16ab53
2 changed files with 101 additions and 2 deletions
@@ -240,9 +240,11 @@ class BandLooper:
            hook = self.tm.layers[self.l0].register_forward_pre_hook(
                swap, with_kwargs=True)
        try:
+            unk = tok.unk_token_id
            eos = [t for t in (tok.eos_token_id,
-                               tok.convert_tokens_to_ids("<end_of_turn>"))
-                   if t is not None and t >= 0]
+                               tok.convert_tokens_to_ids("<end_of_turn>"),
+                               tok.convert_tokens_to_ids("<turn|>"))
+                   if t is not None and t >= 0 and t != unk]
            out = self.model.generate(
                input_ids=input_ids, attention_mask=attention_mask,
                max_new_tokens=max_new_tokens, do_sample=False,