nils/jspace
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

best-of-3 oracle/deployable split: realistic selector loses the overall edge (55.0/27.3 vs oracle 57.8/34.5); paper economics rewritten

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nils
2026-07-14 17:16:00 +02:00
co-authored by Claude Fable 5
parent e5fc453005
commit 2ce646d175
6 changed files with 5212 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/stats_final.py
+39
View File
@@ -200,6 +200,45 @@ def main():
lines.append(f"- {name}: A-only {r['a_only']}, B-only {r['b_only']}, "
f"n={r['n']}, p={r['p']:.4g} ({sig})")
# ---------- 2b. token baselines, per-item (oracle vs deployable) ------
try:
d = json.load(open(OUT / "eval_bestof3_select.json"))
sel = {it["task_id"]: it["selected"] for it in d["per_item"]}
orc = {it["task_id"]: it["oracle"] for it in d["per_item"]}
d2 = json.load(open(OUT / "eval_bestof3_pi.json"))
orc_pi = {it["task_id"]: it["ok"] for it in d2["per_item"]}
lines += ["", "## Token baselines (paired, per-item)", ""]
for name, m in [("best-of-3 ORACLE (any-pass)", orc_pi),
("best-of-3 oracle (selector run)", orc),
("best-of-3 DEPLOYABLE (logprob-selected)", sel)]:
o, h = acc_ci(m), acc_ci(m, mbpp_hard)
lines.append(f"- **{name}**: overall {fmt(o)}; hard {fmt(h)}")
report[name] = {"overall": o, "hard": h}
for base_name, m in [("oracle", orc_pi), ("deployable", sel)]:
for arm_name, arm in [("loop k=4", loop4), ("distill k=1", dist1)]:
for sub, subn in [(None, "overall"), (mbpp_hard, "hard")]:
r = mcnemar(pair(arm, m, sub))
report["mcnemar"][f"bo3-{base_name} vs {arm_name}, {subn}"] = r
sig = "**significant**" if r["p"] < 0.05 else "n.s."
lines.append(f"- bo3-{base_name} vs {arm_name}, {subn}: "
f"arm-only {r['a_only']}, bo3-only "
f"{r['b_only']}, p={r['p']:.4g} ({sig})")
d3 = json.load(open(OUT / "eval_budgetcot_pi.json"))
bc = {it["task_id"]: it["ok"] for it in d3["per_item"]}
o, h = acc_ci(bc), acc_ci(bc, mbpp_hard)
lines.append(f"- **budget-CoT-50 (per-item rerun)**: overall {fmt(o)}; "
f"hard {fmt(h)}")
for arm_name, arm in [("loop k=4", loop4), ("distill k=1", dist1)]:
for sub, subn in [(None, "overall"), (mbpp_hard, "hard")]:
r = mcnemar(pair(arm, bc, sub))
report["mcnemar"][f"budget-cot vs {arm_name}, {subn}"] = r
sig = "**significant**" if r["p"] < 0.05 else "n.s."
lines.append(f"- budget-CoT vs {arm_name}, {subn}: arm-only "
f"{r['a_only']}, cot-only {r['b_only']}, "
f"p={r['p']:.4g} ({sig})")
except FileNotFoundError as e:
lines.append(f"(token-baseline per-item files incomplete: {e})")
# ---------- 3. pooled hard bucket across benchmarks ----------
lines += ["", "## Pooled hard bucket (MBPP + HumanEval + Rust)",
"", "Paired within-item k>0 vs k=0, counts pooled across "