nils/jspace
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

panel: chat-template MC scoring (raw scoring is chance on -it model), parquet dataset mirrors, incremental dump

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nils
2026-07-14 16:00:17 +02:00
co-authored by Claude Fable 5
parent 87308cfb0b
commit 01fd272028
1 changed files with 20 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/eval_mc_panel.py
+20 -10
View File
@@ -37,14 +37,19 @@ def get_items(name):
r["choices"]["label"].index(r["answerKey"]))
for r in ds if r["answerKey"] in r["choices"]["label"]]
elif name == "winogrande":
ds = load_dataset("winogrande", "winogrande_xl", split="validation",
trust_remote_code=True)
items = [(r["sentence"], [r["option1"], r["option2"]],
int(r["answer"]) - 1) for r in ds]
# parquet mirror; cloze -> shared-prefix continuation scoring
ds = load_dataset("allenai/winogrande", "winogrande_xl",
split="validation")
items = []
for r in ds:
pre, _, suf = r["sentence"].partition("_")
items.append(("Complete the sentence: " + pre.rstrip(),
[f"{r[o]}{suf}" for o in ("option1", "option2")],
int(r["answer"]) - 1))
elif name == "hellaswag":
ds = load_dataset("hellaswag", split="validation",
trust_remote_code=True)
items = [(r["ctx"], r["endings"], int(r["label"])) for r in ds]
ds = load_dataset("Rowan/hellaswag", split="validation")
items = [("Complete the sentence: " + r["ctx"], r["endings"],
int(r["label"])) for r in ds]
elif name == "mmlu":
ds = load_dataset("cais/mmlu", "all", split="test")
items = [(r["question"], r["choices"], r["answer"]) for r in ds]
@@ -57,8 +62,12 @@ def get_items(name):
@torch.no_grad()
def score_item(looper, adapter, tok, q, options, k, feedforward):
prompt = q if q.endswith((" ", "\n")) else q + " "
p_ids = tok(prompt, add_special_tokens=True)["input_ids"]
# chat template is load-bearing for the -it model: raw completion
# scoring is chance-level on ARC, template recovers ~0.50
prompt = tok.apply_chat_template([{"role": "user", "content": q}],
tokenize=False,
add_generation_prompt=True)
p_ids = tok(prompt, add_special_tokens=False)["input_ids"]
seqs = [p_ids + tok(o, add_special_tokens=False)["input_ids"]
for o in options]
T = max(len(s) for s in seqs)
@@ -112,7 +121,8 @@ def main():
res[b] = ok / len(items)
print(f"[{args.tag}] {b}: {res[b]:.4f} (n={len(items)}, "
f"{time.time()-t0:.0f}s)", flush=True)
json.dump(res, open(OUT / f"eval_panel_{args.tag}.json", "w"), indent=1)
json.dump(res, open(OUT / f"eval_panel_{args.tag}.json", "w"),
indent=1)
print("wrote", OUT / f"eval_panel_{args.tag}.json")