panel: chat-template MC scoring (raw scoring is chance on -it model), parquet dataset mirrors, incremental dump

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

scripts/eval_mc_panel.py

@@ -37,14 +37,19 @@ def get_items(name):
                   r["choices"]["label"].index(r["answerKey"]))
                  for r in ds if r["answerKey"] in r["choices"]["label"]]
     elif name == "winogrande":
-        ds = load_dataset("winogrande", "winogrande_xl", split="validation",
+        # parquet mirror; cloze -> shared-prefix continuation scoring
-                          trust_remote_code=True)
+        ds = load_dataset("allenai/winogrande", "winogrande_xl",
-        items = [(r["sentence"], [r["option1"], r["option2"]],
+                          split="validation")
-                  int(r["answer"]) - 1) for r in ds]
+        items = []
+        for r in ds:
+            pre, _, suf = r["sentence"].partition("_")
+            items.append(("Complete the sentence: " + pre.rstrip(),
+                          [f"{r[o]}{suf}" for o in ("option1", "option2")],
+                          int(r["answer"]) - 1))
     elif name == "hellaswag":
-        ds = load_dataset("hellaswag", split="validation",
+        ds = load_dataset("Rowan/hellaswag", split="validation")
-                          trust_remote_code=True)
+        items = [("Complete the sentence: " + r["ctx"], r["endings"],
-        items = [(r["ctx"], r["endings"], int(r["label"])) for r in ds]
+                  int(r["label"])) for r in ds]
     elif name == "mmlu":
         ds = load_dataset("cais/mmlu", "all", split="test")
         items = [(r["question"], r["choices"], r["answer"]) for r in ds]
@@ -57,8 +62,12 @@ def get_items(name):
 
 @torch.no_grad()
 def score_item(looper, adapter, tok, q, options, k, feedforward):
-    prompt = q if q.endswith((" ", "\n")) else q + " "
+    # chat template is load-bearing for the -it model: raw completion
-    p_ids = tok(prompt, add_special_tokens=True)["input_ids"]
+    # scoring is chance-level on ARC, template recovers ~0.50
+    prompt = tok.apply_chat_template([{"role": "user", "content": q}],
+                                     tokenize=False,
+                                     add_generation_prompt=True)
+    p_ids = tok(prompt, add_special_tokens=False)["input_ids"]
     seqs = [p_ids + tok(o, add_special_tokens=False)["input_ids"]
             for o in options]
     T = max(len(s) for s in seqs)
@@ -112,7 +121,8 @@ def main():
         res[b] = ok / len(items)
         print(f"[{args.tag}] {b}: {res[b]:.4f} (n={len(items)}, "
               f"{time.time()-t0:.0f}s)", flush=True)
-    json.dump(res, open(OUT / f"eval_panel_{args.tag}.json", "w"), indent=1)
+        json.dump(res, open(OUT / f"eval_panel_{args.tag}.json", "w"),
+                  indent=1)
     print("wrote", OUT / f"eval_panel_{args.tag}.json")