CPU endgame: stats pass (Wilson/McNemar/pooled hard), final figures, PAPER.md rewrite around amortizable-content thesis
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nils
Claude Fable 5
@@ -0,0 +1,250 @@
|
||||
"""Final paper figures (CPU only), regenerated from consolidated results.
|
||||
|
||||
fig_placement.png : band-entrance cliff + structural nulls
|
||||
fig_ladder.png : MBPP-E2B attribution ladder, hard bucket, Wilson CIs
|
||||
fig_scale.png : cross-scale / cross-task attribution grid
|
||||
fig_transfer.png : substrate transfer panel (HumanEval / Rust / BW)
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
OUT = ROOT / "results-loop"
|
||||
|
||||
BLUE, GREEN, ORANGE, GRAY, RED = ("#2b6cb0", "#2f855a", "#dd6b20",
|
||||
"#8a8f98", "#c53030")
|
||||
|
||||
|
||||
def wilson(c, n, z=1.96):
|
||||
p = c / n
|
||||
d = 1 + z * z / n
|
||||
ctr = (p + z * z / (2 * n)) / d
|
||||
hw = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
|
||||
return ctr - hw, ctr + hw
|
||||
|
||||
|
||||
def ev(path, k):
|
||||
d = json.load(open(path))
|
||||
v = d["ks"][str(k)]
|
||||
return v["acc"], v["by_label"].get("hard", float("nan"))
|
||||
|
||||
|
||||
def best_hard(path, exclude0=True):
|
||||
d = json.load(open(path))
|
||||
items = [(int(k), v) for k, v in d["ks"].items()
|
||||
if not (exclude0 and k == "0")]
|
||||
k, v = max(items, key=lambda kv: kv[1]["by_label"].get("hard", 0))
|
||||
return k, v["acc"], v["by_label"].get("hard", 0)
|
||||
|
||||
|
||||
def style(ax):
|
||||
ax.grid(True, color="#e8e8e8", lw=0.7)
|
||||
ax.set_axisbelow(True)
|
||||
for s in ("top", "right"):
|
||||
ax.spines[s].set_visible(False)
|
||||
|
||||
|
||||
# ---------------- fig 1: placement ----------------
|
||||
def fig_placement():
|
||||
entr = [] # (entrance_layer, overall@bestk, hard@bestk)
|
||||
for lo in (9, 11, 12, 13, 14):
|
||||
f = ROOT / f"results-band-{lo}_30" / f"eval_code_band_{lo}_30.json"
|
||||
if lo == 14:
|
||||
f = OUT / "eval_code_code_s0_full.json"
|
||||
k, a, h = best_hard(f)
|
||||
entr.append((lo, a, h))
|
||||
base_a, base_h = ev(OUT / "eval_code_code_s0_full.json", 0)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(6.4, 4.2))
|
||||
style(ax)
|
||||
xs = [e[0] for e in entr]
|
||||
ax.plot(xs, [e[2] for e in entr], "-s", color=BLUE, lw=2, ms=6,
|
||||
label="hard bucket (best k)")
|
||||
ax.plot(xs, [e[1] for e in entr], "-o", color=GRAY, lw=2, ms=5,
|
||||
label="overall (same k)")
|
||||
ax.axhline(base_h, color=BLUE, lw=1, ls=":", alpha=0.6)
|
||||
ax.axhline(base_a, color=GRAY, lw=1, ls=":", alpha=0.6)
|
||||
ax.annotate("base hard", (9.1, base_h + 0.012), fontsize=8, color=BLUE)
|
||||
ax.annotate("base overall", (9.1, base_a + 0.012), fontsize=8, color=GRAY)
|
||||
ax.axvspan(13.5, 14.5, color="#ebf4ff", zorder=0)
|
||||
ax.annotate("lens-identified\nworkspace entrance", (13.55, 0.60),
|
||||
fontsize=8, color=BLUE)
|
||||
ax.set_xticks(xs)
|
||||
ax.set_xlabel("loop entrance layer (exit fixed at L30)")
|
||||
ax.set_ylabel("MBPP pass@1")
|
||||
ax.set_title("Placement cliff: the retrofit works only at the "
|
||||
"lens boundary (L14)", fontsize=11)
|
||||
ax.legend(fontsize=8, frameon=False, loc="center left")
|
||||
fig.text(0.13, 0.005,
|
||||
"Entrances 17/24 (not shown): structurally null — KV sharing "
|
||||
"makes k>0 bit-identical to k=0.", fontsize=7.5, color="#666")
|
||||
fig.tight_layout(rect=(0, 0.03, 1, 1))
|
||||
fig.savefig(OUT / "fig_placement.png", dpi=140, facecolor="white",
|
||||
bbox_inches="tight")
|
||||
print("wrote fig_placement.png")
|
||||
|
||||
|
||||
# ---------------- fig 2: attribution ladder ----------------
|
||||
def fig_ladder():
|
||||
NH = 55
|
||||
rows = [] # (label, hard_acc, n, color, note)
|
||||
|
||||
def add(label, h, n, color, note=""):
|
||||
rows.append((label, h, n, color, note))
|
||||
|
||||
_, h0 = ev(OUT / "eval_code_code_s0_full.json", 0)
|
||||
add("base (k=0, exact)", h0, NH, GRAY)
|
||||
# untrained E2B control (250-item era, n_hard=28)
|
||||
d = json.load(open(OUT / "eval_code_untrained.json"))
|
||||
hu = max(v["by_label"]["hard"] for k, v in d["ks"].items() if k != "0")
|
||||
add("untrained loop (best k)", hu, 28, GRAY)
|
||||
d = json.load(open(OUT / "eval_code_ff.json"))
|
||||
add("trained FF (no recurrence)", d["ks"]["1"]["by_label"]["hard"], 28,
|
||||
ORANGE)
|
||||
_, hp = ev(OUT / "eval_code_pause16.json", 1)
|
||||
add("pause-16 registers (width)", hp, NH, ORANGE)
|
||||
_, hl = ev(OUT / "eval_code_code_s0_full.json", 4)
|
||||
add("loop k=4 (depth)", hl, NH, BLUE, "seed mean 0.375 ± 0.055")
|
||||
d = json.load(open(OUT / "eval_rung2_s1.json"))
|
||||
add("rung-2: + band LoRA (k=4)", d["4"]["by_label"]["hard"], 28, BLUE)
|
||||
_, hd = ev(OUT / "eval_code_distill_s1.json", 1)
|
||||
add("plan-distilled FF", hd, NH, GREEN, "8-run mean 0.457 ± 0.046")
|
||||
d = json.load(open(OUT / "eval_budgetcot.json"))
|
||||
add("budget-CoT (50 visible tok)", d["by_label"]["hard"], NH, "#805ad5")
|
||||
d = json.load(open(OUT / "eval_bestof3.json"))
|
||||
add("best-of-3 sampling (~matched FLOPs)", d["by_label"]["hard"], NH,
|
||||
"#805ad5")
|
||||
d = json.load(open(OUT / "eval_plan_baseline.json"))
|
||||
add("explicit plan in context (ceiling)", d["by_label"]["hard"], NH,
|
||||
"#1a202c")
|
||||
|
||||
fig, ax = plt.subplots(figsize=(7.4, 4.8))
|
||||
style(ax)
|
||||
ys = range(len(rows))[::-1]
|
||||
for y, (label, h, n, color, note) in zip(ys, rows):
|
||||
lo, hi = wilson(round(h * n), n)
|
||||
ax.barh(y, h, color=color, height=0.62, alpha=0.88)
|
||||
ax.plot([lo, hi], [y, y], color="#333", lw=1.2)
|
||||
txt = f"{h:.2f}"
|
||||
if note:
|
||||
txt += f" ({note})"
|
||||
ax.text(hi + 0.015, y, txt, va="center", fontsize=8)
|
||||
ax.set_yticks(list(ys))
|
||||
ax.set_yticklabels([r[0] for r in rows], fontsize=9)
|
||||
ax.set_xlim(0, 1.02)
|
||||
ax.set_xlabel("pass@1, MBPP hard bucket (plan-dependent items)")
|
||||
ax.set_title("Attribution ladder: what closes the plan gap "
|
||||
"(bars: point estimate, whiskers: Wilson 95%)", fontsize=11)
|
||||
fig.tight_layout()
|
||||
fig.savefig(OUT / "fig_ladder.png", dpi=140, facecolor="white",
|
||||
bbox_inches="tight")
|
||||
print("wrote fig_ladder.png")
|
||||
|
||||
|
||||
# ---------------- fig 3: cross-scale grid ----------------
|
||||
def fig_scale():
|
||||
N2 = ROOT / "results-node2-final/results-12b"
|
||||
N1 = ROOT / "results-node-final/results-12b"
|
||||
panels = {
|
||||
("MBPP", "E2B"): [
|
||||
("base", *ev(OUT / "eval_code_code_s0_full.json", 0)),
|
||||
("loop k=4", *ev(OUT / "eval_code_code_s0_full.json", 4)),
|
||||
("adaptive k=4", *ev(OUT / "eval_code_e2b_adaptive.json", 4)),
|
||||
("distill", *ev(OUT / "eval_code_distill_s1.json", 1)),
|
||||
],
|
||||
("MBPP", "12B"): [
|
||||
("base", *ev(N1 / "eval_code_12b_trained.json", 0)),
|
||||
("loop k=4 (α=.3)", *ev(N1 / "eval_code_12b_trained.json", 4)),
|
||||
("adaptive k=4", *ev(N2 / "eval_code_12b_adaptive.json", 4)),
|
||||
("distill", *ev(OUT / "eval_code_12b_distill.json", 1)),
|
||||
],
|
||||
("GSM8K", "E2B"): [
|
||||
("base", *ev(OUT / "eval_uni.json", 0)),
|
||||
("loop k=2", *ev(OUT / "eval_uni.json", 2)),
|
||||
("adaptive k=2", *ev(OUT / "eval_gsm_e2b_adaptive.json", 2)),
|
||||
("distill", *ev(OUT / "eval_gsm_distill_retry.json", 1)),
|
||||
],
|
||||
("GSM8K", "12B"): [
|
||||
("base", *ev(N1 / "eval_12b_gsm_loop.json", 0)),
|
||||
("loop k=2 (α=.3)", *ev(N1 / "eval_12b_gsm_loop.json", 2)),
|
||||
("adaptive k=2", *ev(OUT / "eval_12b_gsm_adaptive.json", 2)),
|
||||
("distill*", *ev(OUT / "eval_12b_gsm_distill.json", 1)),
|
||||
],
|
||||
}
|
||||
fig, axes = plt.subplots(2, 2, figsize=(9.6, 6.6))
|
||||
for ax, ((task, scale), arms) in zip(axes.flat, panels.items()):
|
||||
if (task, scale) == ("GSM8K", "12B"):
|
||||
ax.annotate("*training collapse (0.00)", (3, 0.03), fontsize=7.5,
|
||||
ha="center", color=RED)
|
||||
style(ax)
|
||||
x = range(len(arms))
|
||||
ax.bar([i - 0.19 for i in x], [a[1] for a in arms], width=0.36,
|
||||
color=GRAY, alpha=0.85, label="overall")
|
||||
ax.bar([i + 0.19 for i in x], [a[2] for a in arms], width=0.36,
|
||||
color=BLUE, alpha=0.85, label="hard")
|
||||
ax.axhline(arms[0][1], color=GRAY, lw=1, ls=":")
|
||||
ax.set_xticks(list(x))
|
||||
ax.set_xticklabels([a[0] for a in arms], fontsize=8)
|
||||
ax.set_title(f"{task} · {scale}", fontsize=10)
|
||||
ax.set_ylim(0, 1.0)
|
||||
if ax is axes.flat[0]:
|
||||
ax.legend(fontsize=8, frameon=False)
|
||||
fig.suptitle("Cross-scale attribution: constant-α destroys the 12B "
|
||||
"substrate; state-dependent α restores it (MBPP) but not "
|
||||
"everywhere", fontsize=11.5)
|
||||
fig.tight_layout(rect=(0, 0, 1, 0.96))
|
||||
fig.savefig(OUT / "fig_scale.png", dpi=140, facecolor="white",
|
||||
bbox_inches="tight")
|
||||
print("wrote fig_scale.png")
|
||||
|
||||
|
||||
# ---------------- fig 4: transfer panel ----------------
|
||||
def fig_transfer():
|
||||
def he_hard(path, k):
|
||||
d = json.load(open(OUT / path))
|
||||
return d["ks"][str(k)]["acc"], d["ks"][str(k)]["hard_acc"]
|
||||
|
||||
groups = [
|
||||
("HumanEval\n(MBPP-trained)", [
|
||||
("base", *he_hard("eval_humaneval_trained.json", 0)),
|
||||
("loop k=4", *he_hard("eval_humaneval_trained.json", 4)),
|
||||
("distill", *he_hard("eval_humaneval_distill_transfer.json", 1)),
|
||||
]),
|
||||
("Rust / MultiPL-E\n(Python-trained)", [
|
||||
("base", *ev(OUT / "eval_rust_py_transfer.json", 0)),
|
||||
("loop k=4", *ev(OUT / "eval_rust_py_transfer.json", 4)),
|
||||
]),
|
||||
]
|
||||
fig, axes = plt.subplots(1, 2, figsize=(7.6, 3.8))
|
||||
for ax, (title, arms) in zip(axes, groups):
|
||||
style(ax)
|
||||
x = range(len(arms))
|
||||
ax.bar([i - 0.19 for i in x], [a[1] for a in arms], width=0.36,
|
||||
color=GRAY, alpha=0.85, label="overall")
|
||||
ax.bar([i + 0.19 for i in x],
|
||||
[a[2] if a[2] is not None else 0 for a in arms],
|
||||
width=0.36, color=BLUE, alpha=0.85, label="hard")
|
||||
ax.set_xticks(list(x))
|
||||
ax.set_xticklabels([a[0] for a in arms], fontsize=8.5)
|
||||
ax.set_title(title, fontsize=9.5)
|
||||
ax.set_ylim(0, 1.0)
|
||||
axes[0].legend(fontsize=8, frameon=False)
|
||||
fig.suptitle("Transfer: the implant moves with the substrate, "
|
||||
"not the task", fontsize=11.5)
|
||||
fig.tight_layout(rect=(0, 0, 1, 0.94))
|
||||
fig.savefig(OUT / "fig_transfer.png", dpi=140, facecolor="white",
|
||||
bbox_inches="tight")
|
||||
print("wrote fig_transfer.png")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fig_placement()
|
||||
fig_ladder()
|
||||
fig_scale()
|
||||
fig_transfer()
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Final statistics pass over all eval per_item logs (CPU only).
|
||||
|
||||
Produces results-loop/STATS.md + stats_final.json:
|
||||
1. Wilson 95% CIs for every headline number (overall + hard bucket).
|
||||
2. Exact McNemar tests for the key paired comparisons (same items).
|
||||
3. Pooled hard bucket across MBPP + HumanEval + Rust (paired k>0 vs k0
|
||||
within each item, pooled counts).
|
||||
4. Label-robustness check: hard bucket redefined via consensus k0
|
||||
failure across seeds instead of the single greedy labeling run.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
OUT = Path(__file__).resolve().parent.parent / "results-loop"
|
||||
|
||||
|
||||
def wilson(c, n, z=1.96):
|
||||
if n == 0:
|
||||
return (float("nan"), float("nan"))
|
||||
p = c / n
|
||||
d = 1 + z * z / n
|
||||
ctr = (p + z * z / (2 * n)) / d
|
||||
hw = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
|
||||
return (ctr - hw, ctr + hw)
|
||||
|
||||
|
||||
def binom_two_sided(k, n):
|
||||
"""Exact two-sided binomial test p-value, p0=0.5 (for McNemar)."""
|
||||
if n == 0:
|
||||
return 1.0
|
||||
def pmf(i):
|
||||
return math.comb(n, i) * 0.5 ** n
|
||||
pk = pmf(k)
|
||||
return min(1.0, sum(pmf(i) for i in range(n + 1) if pmf(i) <= pk + 1e-12))
|
||||
|
||||
|
||||
def mcnemar(pairs):
|
||||
"""pairs: list of (a_ok, b_ok). Returns dict with discordants + p."""
|
||||
b01 = sum(1 for a, b in pairs if not a and b) # b wins
|
||||
b10 = sum(1 for a, b in pairs if a and not b) # a wins
|
||||
return {"n": len(pairs), "a_only": b10, "b_only": b01,
|
||||
"p": binom_two_sided(min(b01, b10), b01 + b10)}
|
||||
|
||||
|
||||
def load_labels(path, key, lab_key="label"):
|
||||
data = json.load(open(OUT / path))
|
||||
return {it[key]: it[lab_key] for it in data
|
||||
if it.get("split", "test") == "test"}
|
||||
|
||||
|
||||
def per_item(fname, k):
|
||||
d = json.load(open(OUT / fname))
|
||||
v = d["ks"][str(k)]
|
||||
key = "task_id" if "task_id" in v["per_item"][0] else "idx"
|
||||
return {it[key]: it["ok"] for it in v["per_item"]}
|
||||
|
||||
|
||||
def acc_ci(ok_map, subset=None):
|
||||
ids = [i for i in ok_map if subset is None or i in subset]
|
||||
c = sum(ok_map[i] for i in ids)
|
||||
lo, hi = wilson(c, len(ids))
|
||||
return {"acc": c / len(ids) if ids else float("nan"), "n": len(ids),
|
||||
"ci": [round(lo, 3), round(hi, 3)]}
|
||||
|
||||
|
||||
def fmt(r):
|
||||
return f"{r['acc']:.3f} [{r['ci'][0]:.3f}, {r['ci'][1]:.3f}] (n={r['n']})"
|
||||
|
||||
|
||||
def main():
|
||||
mbpp_lab = load_labels("mbpp_data.json", "task_id")
|
||||
he_lab = {k: ("hard" if v else "easy") # plan-reachable flag; hard needs k0-fail
|
||||
for k, v in json.load(open(OUT / "humaneval_labels.json")).items()}
|
||||
rust_lab = load_labels("rust_data.json", "task_id")
|
||||
|
||||
mbpp_hard = {t for t, l in mbpp_lab.items() if l == "hard"}
|
||||
rust_hard = {t for t, l in rust_lab.items() if l == "hard"}
|
||||
|
||||
report = {}
|
||||
lines = ["# Final statistics pass", ""]
|
||||
|
||||
# ---------- 1. headline numbers with Wilson CIs ----------
|
||||
lines += ["## Headline numbers (Wilson 95% CIs)", ""]
|
||||
ARMS = [
|
||||
# (label, file, k, hard-subset)
|
||||
("MBPP loop s0 k=0 (base)", "eval_code_code_s0_full.json", 0, mbpp_hard),
|
||||
("MBPP loop s0 k=2", "eval_code_code_s0_full.json", 2, mbpp_hard),
|
||||
("MBPP loop s0 k=4", "eval_code_code_s0_full.json", 4, mbpp_hard),
|
||||
("MBPP distill s1 k=1 (FF)", "eval_code_distill_s1.json", 1, mbpp_hard),
|
||||
("MBPP pause16 k=1", "eval_code_pause16.json", 1, mbpp_hard),
|
||||
("MBPP stack-train k=4", "eval_code_stack_train.json", 4, mbpp_hard),
|
||||
("MBPP distill-in-loopmode k=2", "eval_code_distill_loopmode.json", 2, mbpp_hard),
|
||||
("Rust transfer k=0", "eval_rust_py_transfer.json", 0, rust_hard),
|
||||
("Rust transfer k=4", "eval_rust_py_transfer.json", 4, rust_hard),
|
||||
]
|
||||
arm_maps = {}
|
||||
for label, f, k, hard in ARMS:
|
||||
m = per_item(f, k)
|
||||
arm_maps[label] = (m, hard)
|
||||
o, h = acc_ci(m), acc_ci(m, hard)
|
||||
report[label] = {"overall": o, "hard": h}
|
||||
lines.append(f"- **{label}**: overall {fmt(o)}; hard {fmt(h)}")
|
||||
|
||||
# HumanEval: hard = plan-reachable AND k0-fail (per its eval definition)
|
||||
he_tr0 = per_item("eval_humaneval_trained.json", 0)
|
||||
he_hard = {t for t in he_tr0 if he_lab.get(t) == "hard" and not he_tr0[t]}
|
||||
for label, f, k in [("HumanEval loop k=4", "eval_humaneval_trained.json", 4),
|
||||
("HumanEval distill k=1", "eval_humaneval_distill_transfer.json", 1)]:
|
||||
m = per_item(f, k)
|
||||
o, h = acc_ci(m), acc_ci(m, he_hard)
|
||||
arm_maps[label] = (m, he_hard)
|
||||
report[label] = {"overall": o, "hard": h}
|
||||
lines.append(f"- **{label}**: overall {fmt(o)}; hard {fmt(h)}")
|
||||
|
||||
# best-of-3 / budget-CoT (no per_item; CI from counts)
|
||||
for label, f in [("MBPP best-of-3 (compute-matched)", "eval_bestof3.json"),
|
||||
("MBPP budget-CoT-50", "eval_budgetcot.json")]:
|
||||
d = json.load(open(OUT / f))
|
||||
n, nh = 500, len(mbpp_hard)
|
||||
o = {"acc": d["acc"], "n": n,
|
||||
"ci": [round(x, 3) for x in wilson(round(d["acc"] * n), n)]}
|
||||
hacc = d["by_label"]["hard"]
|
||||
h = {"acc": hacc, "n": nh,
|
||||
"ci": [round(x, 3) for x in wilson(round(hacc * nh), nh)]}
|
||||
report[label] = {"overall": o, "hard": h}
|
||||
lines.append(f"- **{label}**: overall {fmt(o)}; hard {fmt(h)}")
|
||||
|
||||
# distill seed spread
|
||||
hs, os_ = [], []
|
||||
files = ["eval_code_code_distill.json"] + [
|
||||
f"eval_code_distill_s{s}.json" for s in range(1, 8)]
|
||||
for f in files:
|
||||
try:
|
||||
m = per_item(f, 1)
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
hs.append(acc_ci(m, mbpp_hard)["acc"])
|
||||
os_.append(acc_ci(m)["acc"])
|
||||
mean = sum(hs) / len(hs)
|
||||
sd = (sum((x - mean) ** 2 for x in hs) / (len(hs) - 1)) ** 0.5
|
||||
lines.append(f"- **MBPP distill, {len(hs)} runs (hard)**: mean {mean:.3f} "
|
||||
f"± {sd:.3f} sd (range {min(hs):.3f}-{max(hs):.3f}); "
|
||||
f"overall mean {sum(os_)/len(os_):.3f}")
|
||||
report["distill_seed_spread"] = {"hard": hs, "overall": os_}
|
||||
|
||||
# loop seed spread (s0..s4 k=4)
|
||||
lh = []
|
||||
for tag in ["s0_full", "s1", "s2", "s3", "s4"]:
|
||||
try:
|
||||
m = per_item(f"eval_code_code_{tag}.json", 4)
|
||||
lh.append(acc_ci(m, mbpp_hard)["acc"])
|
||||
except (FileNotFoundError, KeyError):
|
||||
pass
|
||||
if lh:
|
||||
mean = sum(lh) / len(lh)
|
||||
sd = (sum((x - mean) ** 2 for x in lh) / max(1, len(lh) - 1)) ** 0.5
|
||||
lines.append(f"- **MBPP loop seeds k=4 (hard)**: mean {mean:.3f} "
|
||||
f"± {sd:.3f} sd (n_seeds={len(lh)})")
|
||||
report["loop_seed_spread_hard"] = lh
|
||||
|
||||
# ---------- 2. McNemar paired tests ----------
|
||||
lines += ["", "## McNemar exact tests (paired on items)", ""]
|
||||
|
||||
def pair(m_a, m_b, subset=None):
|
||||
ids = [i for i in m_a if i in m_b
|
||||
and (subset is None or i in subset)]
|
||||
return [(m_a[i], m_b[i]) for i in ids]
|
||||
|
||||
loop0, _ = arm_maps["MBPP loop s0 k=0 (base)"]
|
||||
loop4, _ = arm_maps["MBPP loop s0 k=4"]
|
||||
dist1, _ = arm_maps["MBPP distill s1 k=1 (FF)"]
|
||||
stack4, _ = arm_maps["MBPP stack-train k=4"]
|
||||
|
||||
TESTS = [
|
||||
("loop k=4 vs k=0, overall", loop0, loop4, None),
|
||||
("loop k=4 vs k=0, hard", loop0, loop4, mbpp_hard),
|
||||
("distill k=1 vs loop k=4, overall", loop4, dist1, None),
|
||||
("distill k=1 vs loop k=4, hard", loop4, dist1, mbpp_hard),
|
||||
("stack-train k=4 vs distill k=1, hard", dist1, stack4, mbpp_hard),
|
||||
("HumanEval loop k=4 vs k=0, overall", he_tr0,
|
||||
arm_maps["HumanEval loop k=4"][0], None),
|
||||
("HumanEval distill k=1 vs k=0, overall", he_tr0,
|
||||
arm_maps["HumanEval distill k=1"][0], None),
|
||||
]
|
||||
rust0 = arm_maps["Rust transfer k=0"][0]
|
||||
rust4 = arm_maps["Rust transfer k=4"][0]
|
||||
TESTS += [("Rust loop k=4 vs k=0, overall", rust0, rust4, None),
|
||||
("Rust loop k=4 vs k=0, hard", rust0, rust4, rust_hard)]
|
||||
|
||||
report["mcnemar"] = {}
|
||||
for name, a, b, subset in TESTS:
|
||||
r = mcnemar(pair(a, b, subset))
|
||||
report["mcnemar"][name] = r
|
||||
sig = "**significant**" if r["p"] < 0.05 else "n.s."
|
||||
lines.append(f"- {name}: A-only {r['a_only']}, B-only {r['b_only']}, "
|
||||
f"n={r['n']}, p={r['p']:.4g} ({sig})")
|
||||
|
||||
# ---------- 3. pooled hard bucket across benchmarks ----------
|
||||
lines += ["", "## Pooled hard bucket (MBPP + HumanEval + Rust)",
|
||||
"", "Paired within-item k>0 vs k=0, counts pooled across "
|
||||
"benchmarks (loop arm; distill pooled where available).", ""]
|
||||
pooled_loop = (pair(loop0, loop4, mbpp_hard)
|
||||
+ pair(he_tr0, arm_maps["HumanEval loop k=4"][0], he_hard)
|
||||
+ pair(rust0, rust4, rust_hard))
|
||||
r = mcnemar(pooled_loop)
|
||||
c_base = sum(a for a, _ in pooled_loop)
|
||||
c_loop = sum(b for _, b in pooled_loop)
|
||||
n = len(pooled_loop)
|
||||
lines.append(f"- **loop**: base {c_base}/{n} -> loop {c_loop}/{n} "
|
||||
f"({c_base/n:.3f} -> {c_loop/n:.3f}, CI "
|
||||
f"{[round(x,3) for x in wilson(c_loop, n)]}), "
|
||||
f"McNemar p={r['p']:.3g}")
|
||||
report["pooled_hard_loop"] = {"base": c_base, "arm": c_loop, "n": n,
|
||||
"mcnemar": r}
|
||||
pooled_dist = (pair(loop0, dist1, mbpp_hard)
|
||||
+ pair(he_tr0, arm_maps["HumanEval distill k=1"][0], he_hard))
|
||||
r = mcnemar(pooled_dist)
|
||||
c_base = sum(a for a, _ in pooled_dist)
|
||||
c_d = sum(b for _, b in pooled_dist)
|
||||
n = len(pooled_dist)
|
||||
lines.append(f"- **distill**: base {c_base}/{n} -> distill {c_d}/{n} "
|
||||
f"({c_base/n:.3f} -> {c_d/n:.3f}, CI "
|
||||
f"{[round(x,3) for x in wilson(c_d, n)]}), "
|
||||
f"McNemar p={r['p']:.3g}")
|
||||
report["pooled_hard_distill"] = {"base": c_base, "arm": c_d, "n": n,
|
||||
"mcnemar": r}
|
||||
|
||||
# ---------- 4. label robustness: consensus-k0 hard set ----------
|
||||
lines += ["", "## Label robustness (consensus-k0 hard set)", "",
|
||||
"Hard bucket redefined as: labeled hard AND k=0 fails in "
|
||||
"every seed's own eval run (removes single-greedy-run "
|
||||
"selection noise).", ""]
|
||||
k0maps = []
|
||||
for tag in ["s0_full", "s1", "s2", "s3", "s4"]:
|
||||
try:
|
||||
k0maps.append(per_item(f"eval_code_code_{tag}.json", 0))
|
||||
except (FileNotFoundError, KeyError):
|
||||
pass
|
||||
consensus = {t for t in mbpp_hard
|
||||
if all(not m.get(t, False) for m in k0maps)}
|
||||
lines.append(f"- consensus hard set: {len(consensus)} of "
|
||||
f"{len(mbpp_hard)} labeled-hard items")
|
||||
for label in ["MBPP loop s0 k=4", "MBPP distill s1 k=1 (FF)",
|
||||
"MBPP stack-train k=4"]:
|
||||
m, _ = arm_maps[label]
|
||||
r0 = acc_ci(m, mbpp_hard)
|
||||
rc = acc_ci(m, consensus)
|
||||
lines.append(f"- {label}: labeled-hard {r0['acc']:.3f} -> "
|
||||
f"consensus-hard {fmt(rc)}")
|
||||
report.setdefault("consensus_hard", {})[label] = rc
|
||||
|
||||
(OUT / "STATS.md").write_text("\n".join(lines) + "\n")
|
||||
json.dump(report, open(OUT / "stats_final.json", "w"), indent=1)
|
||||
print("\n".join(lines))
|
||||
print("\nwrote", OUT / "STATS.md", "and stats_final.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user