kcurves: full design-space grid (10 panels, distill reference line, auto-fills pending arms)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

scripts/fig_kcurves.py

@@ -1,6 +1,8 @@
 """Small-multiples: pass@1 vs loop depth k, easy (pastel) + hard (dark),
-one panel per regime arm. All on the same 250-item MBPP eval, e400.
-Per-depth cells fall back to log-parsed values until its JSON lands.
+one panel per regime arm — the full design-space sweep. All loop arms on
+the same 250-item MBPP eval, e400 checkpoints. Final panel: no-recurrence
+references (FF control, plan-distill; distill on the 500-item set).
+Pending arms render as empty boxes and fill in on re-render.
 """
 import json
 import re
@@ -11,16 +13,15 @@ matplotlib.use("Agg")
 import matplotlib.pyplot as plt
 
 OUT = Path(__file__).resolve().parent.parent / "results-loop"
+LOGD = Path("/home/nils/gpuq/spark-gpu0")
 
 def cells(fname):
     try:
         d = json.load(open(OUT / fname))
     except FileNotFoundError:
         return None
-    out = {}
-    for k, v in d["ks"].items():
-        out[int(k)] = (v["by_label"].get("easy"), v["by_label"].get("hard"))
-    return out
+    return {int(k): (v["by_label"].get("easy"), v["by_label"].get("hard"))
+            for k, v in d["ks"].items()}
 
 def cells_from_log(logfile):
     out = {}
@@ -36,48 +37,74 @@ def cells_from_log(logfile):
     return out or None
 
 PANELS = [
-    ("untrained α-merge\n(training-free, ρ=0.3)",
-     cells("eval_code_untrained.json"), "#2b6cb0"),
-    ("trained merge\n(fixed A,B · curriculum)",
+    ("untrained α-merge\ntraining-free · ρ=0.3",
+     cells("eval_code_untrained.json"), "#8a8f98"),
+    ("trained merge  ★\nfixed A,B · curriculum",
      cells("eval_code_trained.json"), "#2b6cb0"),
-    ("unconstrained rec\n(learned A,B · noise h₀ · rand-k) ρ→4.5",
+    ("tied-alpha\nlearned a, B=(1−a) tied · curriculum",
+     cells("eval_code_ta_e400.json"), "#0987a0"),
+    ("random-depth merge\nfixed A,B · NO curriculum",
+     cells("eval_code_merge_rk16_e400.json")
+     or cells_from_log(LOGD / "zzz_a_randk.log"), "#3182ce"),
+    ("unconstrained rec\nlearned A,B · noise h₀ · ρ→4.5",
      cells("eval_code_rec16_partial.json"), "#c53030"),
-    ("Parcae rec\n(ρ<1 enforced · learned B · noise h₀) ρ→0.29",
+    ("Parcae rec\nρ<1 enforced · learned B · ρ→0.29",
      cells("eval_code_parcae16_e400.json"), "#2f855a"),
-    ("per-depth merges\n(time-varying · anchored · curriculum)",
+    ("per-depth merges\ntime-varying · anchored",
      cells("eval_code_pd4_e400.json")
-     or cells_from_log("/home/nils/gpuq/spark-gpu0/perdepth_arm.log"),
-     "#805ad5"),
+     or cells_from_log(LOGD / "perdepth_arm.log"), "#805ad5"),
+    ("noise-s₀ merge\nfixed A,B · curriculum · noise s₀",
+     cells("eval_code_merge_ns_e400.json")
+     or cells_from_log(LOGD / "zzz_b_noises0.log"), "#b83280"),
+    ("4× capacity merge\nMLP h=2048 · curriculum",
+     cells("eval_code_merge_h2048_e400.json")
+     or cells_from_log(LOGD / "zzz_c_h2048.log"), "#5f6b7a"),
+    ("Parcae rec, seed 1",
+     cells("eval_code_parcae16_s1_e400.json")
+     or cells_from_log(LOGD / "zzz_d_parcae_s1.log"), "#276749"),
 ]
 
-fig, axes = plt.subplots(1, 5, figsize=(15.5, 3.6), sharey=True)
-for ax, (title, kc, dark) in zip(axes, PANELS):
+fig, axes = plt.subplots(2, 5, figsize=(16, 6.6), sharey=True)
+axf = axes.flatten()
+for ax, (title, kc, dark) in zip(axf, PANELS):
     ax.grid(True, color="#ececec", lw=0.7)
     ax.set_axisbelow(True)
     for s in ("top", "right"):
         ax.spines[s].set_visible(False)
+    ax.set_title(title, fontsize=8.6)
+    ax.set_ylim(0, 1.02)
     if not kc:
-        ax.set_title(title + "\n(pending)", fontsize=9)
+        ax.text(0.5, 0.5, "pending", transform=ax.transAxes, ha="center",
+                color="#999", fontsize=10)
         continue
     ks = sorted(kc)
     xs = range(len(ks))
-    easy = [kc[k][0] for k in ks]
-    hard = [kc[k][1] for k in ks]
-    # pastel = dark color at low alpha via manual blend to white
-    ax.plot(xs, easy, "-o", color=dark, alpha=0.32, lw=2.4, ms=5,
-            label="easy")
-    ax.plot(xs, hard, "-s", color=dark, lw=2.4, ms=5, label="hard")
+    ax.plot(xs, [kc[k][0] for k in ks], "-o", color=dark, alpha=0.32,
+            lw=2.2, ms=4.5)
+    ax.plot(xs, [kc[k][1] for k in ks], "-s", color=dark, lw=2.2, ms=4.5)
+    # references: base levels (dotted) and distill hard (dashed amber)
     ax.axhline(kc[ks[0]][0], color="#999", lw=0.8, ls=":")
     ax.axhline(kc[ks[0]][1], color="#999", lw=0.8, ls=":")
+    ax.axhline(0.457, color="#b7791f", lw=1.0, ls="--", alpha=0.7)
     ax.set_xticks(list(xs))
-    ax.set_xticklabels([str(k) for k in ks], fontsize=8)
-    ax.set_title(title, fontsize=9)
-    ax.set_xlabel("k", fontsize=9)
-    ax.set_ylim(0, 1.02)
-axes[0].set_ylabel("pass@1 (easy pastel / hard dark)")
-axes[0].legend(fontsize=8, frameon=False, loc="center right")
-fig.suptitle("Depth curves by regime — same frozen band, data, and 250-item "
-             "eval; dotted lines = k=0 base levels", fontsize=11, y=1.04)
+    ax.set_xticklabels([str(k) for k in ks], fontsize=7.5)
+    ax.set_xlabel("k", fontsize=8)
+for i in (0, 5):
+    axf[i].set_ylabel("pass@1", fontsize=9)
+
+from matplotlib.lines import Line2D
+axf[0].legend(handles=[
+    Line2D([], [], color="#555", alpha=0.32, marker="o", lw=2.2, label="easy"),
+    Line2D([], [], color="#555", marker="s", lw=2.2, label="hard"),
+    Line2D([], [], color="#b7791f", ls="--", lw=1.0,
+           label="distill hard (45.7%, no loop)"),
+    Line2D([], [], color="#999", ls=":", lw=0.8, label="k=0 base levels"),
+], fontsize=7, frameon=False, loc="center right")
+
+fig.suptitle("The design space, measured — depth curves for every regime "
+             "(same frozen band, data, 250-item eval; ★ = recommended recipe; "
+             "amber dashes = what distillation reaches with no loop at all)",
+             fontsize=11, y=1.0)
 fig.tight_layout()
 fig.savefig(OUT / "fig_kcurves.png", dpi=140, facecolor="white",
             bbox_inches="tight")