diff --git a/scripts/eval_mc_panel.py b/scripts/eval_mc_panel.py index b175b95..7fbc76b 100644 --- a/scripts/eval_mc_panel.py +++ b/scripts/eval_mc_panel.py @@ -37,14 +37,19 @@ def get_items(name): r["choices"]["label"].index(r["answerKey"])) for r in ds if r["answerKey"] in r["choices"]["label"]] elif name == "winogrande": - ds = load_dataset("winogrande", "winogrande_xl", split="validation", - trust_remote_code=True) - items = [(r["sentence"], [r["option1"], r["option2"]], - int(r["answer"]) - 1) for r in ds] + # parquet mirror; cloze -> shared-prefix continuation scoring + ds = load_dataset("allenai/winogrande", "winogrande_xl", + split="validation") + items = [] + for r in ds: + pre, _, suf = r["sentence"].partition("_") + items.append(("Complete the sentence: " + pre.rstrip(), + [f"{r[o]}{suf}" for o in ("option1", "option2")], + int(r["answer"]) - 1)) elif name == "hellaswag": - ds = load_dataset("hellaswag", split="validation", - trust_remote_code=True) - items = [(r["ctx"], r["endings"], int(r["label"])) for r in ds] + ds = load_dataset("Rowan/hellaswag", split="validation") + items = [("Complete the sentence: " + r["ctx"], r["endings"], + int(r["label"])) for r in ds] elif name == "mmlu": ds = load_dataset("cais/mmlu", "all", split="test") items = [(r["question"], r["choices"], r["answer"]) for r in ds] @@ -57,8 +62,12 @@ def get_items(name): @torch.no_grad() def score_item(looper, adapter, tok, q, options, k, feedforward): - prompt = q if q.endswith((" ", "\n")) else q + " " - p_ids = tok(prompt, add_special_tokens=True)["input_ids"] + # chat template is load-bearing for the -it model: raw completion + # scoring is chance-level on ARC, template recovers ~0.50 + prompt = tok.apply_chat_template([{"role": "user", "content": q}], + tokenize=False, + add_generation_prompt=True) + p_ids = tok(prompt, add_special_tokens=False)["input_ids"] seqs = [p_ids + tok(o, add_special_tokens=False)["input_ids"] for o in options] T = max(len(s) for s in seqs) @@ -112,7 +121,8 @@ def main(): res[b] = ok / len(items) print(f"[{args.tag}] {b}: {res[b]:.4f} (n={len(items)}, " f"{time.time()-t0:.0f}s)", flush=True) - json.dump(res, open(OUT / f"eval_panel_{args.tag}.json", "w"), indent=1) + json.dump(res, open(OUT / f"eval_panel_{args.tag}.json", "w"), + indent=1) print("wrote", OUT / f"eval_panel_{args.tag}.json")