NilsandClaude Fable 5
ef9c08966c
Reproduction of the 2026 workspace/J-lens paper on gemma-4 (E2B/12B/26B), plus the workspace-loop retrofit line: merge adapter, prompt-only latent planning (MBPP), carry variant, attribution controls (FF/pause/untrained), band-location ablation, Blocksworld harness, 12B replication scripts. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
52 lines
2.1 KiB
Python
52 lines
2.1 KiB
Python
"""MBPP latent-planning results: pass@1 vs loop depth, trained vs untrained."""
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import matplotlib
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt
|
|
|
|
OUT = Path(__file__).resolve().parent.parent / "results-loop"
|
|
tr = json.load(open(OUT / "eval_code_trained.json"))
|
|
un = json.load(open(OUT / "eval_code_untrained.json"))
|
|
ks = sorted(int(k) for k in tr["ks"])
|
|
BLUE, GRAY = "#2b6cb0", "#8a8f98"
|
|
|
|
def curve(res, sel):
|
|
return [res["ks"][str(k)]["acc"] if sel == "all"
|
|
else res["ks"][str(k)]["by_label"].get(sel, 0.0) for k in ks]
|
|
|
|
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 4.2))
|
|
for ax in (ax1, ax2):
|
|
ax.grid(True, color="#e5e5e5", lw=0.7)
|
|
ax.set_axisbelow(True)
|
|
for s in ("top", "right"):
|
|
ax.spines[s].set_visible(False)
|
|
ax.set_xticks(ks)
|
|
ax.set_xlabel("loop depth k (prompt-only)")
|
|
|
|
ax1.plot(ks, curve(tr, "all"), "-o", color=BLUE, lw=2, ms=5, label="trained · all")
|
|
ax1.plot(ks, curve(un, "all"), "-o", color=GRAY, lw=2, ms=5, label="untrained · all")
|
|
ax1.axhline(curve(tr, "all")[0], color="#bbb", lw=1, ls=":")
|
|
ax1.set_ylabel("pass@1 (250 MBPP test items)")
|
|
ax1.set_title("Overall: trained k=2 beats no-loop baseline", fontsize=11)
|
|
ax1.legend(fontsize=8, frameon=False)
|
|
|
|
ax2.plot(ks, curve(tr, "hard"), "-s", color=BLUE, lw=2, ms=5,
|
|
label="trained · hard (plan-only)")
|
|
ax2.plot(ks, curve(un, "hard"), "-s", color=GRAY, lw=2, ms=5,
|
|
label="untrained · hard")
|
|
ax2.set_ylabel("pass@1, hard bucket (~56 items)")
|
|
ax2.set_title("Planning-dependent problems: 3.6% → 46.4%", fontsize=11)
|
|
ax2.legend(fontsize=8, frameon=False)
|
|
ax2.annotate("silent loops recover ~46%\nof explicit-planning gap",
|
|
xy=(4, curve(tr, "hard")[-1]), xytext=(1.8, 0.30), fontsize=8,
|
|
color=BLUE, arrowprops=dict(arrowstyle="-", color=BLUE, lw=0.8))
|
|
|
|
fig.suptitle("MBPP latent planning: loop the workspace over the prompt, "
|
|
"then write code normally", fontsize=12, y=1.02)
|
|
fig.tight_layout()
|
|
fig.savefig(OUT / "loop_eval_code.png", dpi=140, bbox_inches="tight",
|
|
facecolor="white")
|
|
print("wrote", OUT / "loop_eval_code.png")
|