diff --git a/results-loop/fig_kcurves.png b/results-loop/fig_kcurves.png index 99fbd61..f2a8c85 100644 Binary files a/results-loop/fig_kcurves.png and b/results-loop/fig_kcurves.png differ diff --git a/scripts/fig_kcurves.py b/scripts/fig_kcurves.py index 490e9ec..b0347a0 100644 --- a/scripts/fig_kcurves.py +++ b/scripts/fig_kcurves.py @@ -1,6 +1,8 @@ """Small-multiples: pass@1 vs loop depth k, easy (pastel) + hard (dark), -one panel per regime arm. All on the same 250-item MBPP eval, e400. -Per-depth cells fall back to log-parsed values until its JSON lands. +one panel per regime arm — the full design-space sweep. All loop arms on +the same 250-item MBPP eval, e400 checkpoints. Final panel: no-recurrence +references (FF control, plan-distill; distill on the 500-item set). +Pending arms render as empty boxes and fill in on re-render. """ import json import re @@ -11,16 +13,15 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt OUT = Path(__file__).resolve().parent.parent / "results-loop" +LOGD = Path("/home/nils/gpuq/spark-gpu0") def cells(fname): try: d = json.load(open(OUT / fname)) except FileNotFoundError: return None - out = {} - for k, v in d["ks"].items(): - out[int(k)] = (v["by_label"].get("easy"), v["by_label"].get("hard")) - return out + return {int(k): (v["by_label"].get("easy"), v["by_label"].get("hard")) + for k, v in d["ks"].items()} def cells_from_log(logfile): out = {} @@ -36,48 +37,74 @@ def cells_from_log(logfile): return out or None PANELS = [ - ("untrained α-merge\n(training-free, ρ=0.3)", - cells("eval_code_untrained.json"), "#2b6cb0"), - ("trained merge\n(fixed A,B · curriculum)", + ("untrained α-merge\ntraining-free · ρ=0.3", + cells("eval_code_untrained.json"), "#8a8f98"), + ("trained merge ★\nfixed A,B · curriculum", cells("eval_code_trained.json"), "#2b6cb0"), - ("unconstrained rec\n(learned A,B · noise h₀ · rand-k) ρ→4.5", + ("tied-alpha\nlearned a, B=(1−a) tied · curriculum", + cells("eval_code_ta_e400.json"), "#0987a0"), + ("random-depth merge\nfixed A,B · NO curriculum", + cells("eval_code_merge_rk16_e400.json") + or cells_from_log(LOGD / "zzz_a_randk.log"), "#3182ce"), + ("unconstrained rec\nlearned A,B · noise h₀ · ρ→4.5", cells("eval_code_rec16_partial.json"), "#c53030"), - ("Parcae rec\n(ρ<1 enforced · learned B · noise h₀) ρ→0.29", + ("Parcae rec\nρ<1 enforced · learned B · ρ→0.29", cells("eval_code_parcae16_e400.json"), "#2f855a"), - ("per-depth merges\n(time-varying · anchored · curriculum)", + ("per-depth merges\ntime-varying · anchored", cells("eval_code_pd4_e400.json") - or cells_from_log("/home/nils/gpuq/spark-gpu0/perdepth_arm.log"), - "#805ad5"), + or cells_from_log(LOGD / "perdepth_arm.log"), "#805ad5"), + ("noise-s₀ merge\nfixed A,B · curriculum · noise s₀", + cells("eval_code_merge_ns_e400.json") + or cells_from_log(LOGD / "zzz_b_noises0.log"), "#b83280"), + ("4× capacity merge\nMLP h=2048 · curriculum", + cells("eval_code_merge_h2048_e400.json") + or cells_from_log(LOGD / "zzz_c_h2048.log"), "#5f6b7a"), + ("Parcae rec, seed 1", + cells("eval_code_parcae16_s1_e400.json") + or cells_from_log(LOGD / "zzz_d_parcae_s1.log"), "#276749"), ] -fig, axes = plt.subplots(1, 5, figsize=(15.5, 3.6), sharey=True) -for ax, (title, kc, dark) in zip(axes, PANELS): +fig, axes = plt.subplots(2, 5, figsize=(16, 6.6), sharey=True) +axf = axes.flatten() +for ax, (title, kc, dark) in zip(axf, PANELS): ax.grid(True, color="#ececec", lw=0.7) ax.set_axisbelow(True) for s in ("top", "right"): ax.spines[s].set_visible(False) + ax.set_title(title, fontsize=8.6) + ax.set_ylim(0, 1.02) if not kc: - ax.set_title(title + "\n(pending)", fontsize=9) + ax.text(0.5, 0.5, "pending", transform=ax.transAxes, ha="center", + color="#999", fontsize=10) continue ks = sorted(kc) xs = range(len(ks)) - easy = [kc[k][0] for k in ks] - hard = [kc[k][1] for k in ks] - # pastel = dark color at low alpha via manual blend to white - ax.plot(xs, easy, "-o", color=dark, alpha=0.32, lw=2.4, ms=5, - label="easy") - ax.plot(xs, hard, "-s", color=dark, lw=2.4, ms=5, label="hard") + ax.plot(xs, [kc[k][0] for k in ks], "-o", color=dark, alpha=0.32, + lw=2.2, ms=4.5) + ax.plot(xs, [kc[k][1] for k in ks], "-s", color=dark, lw=2.2, ms=4.5) + # references: base levels (dotted) and distill hard (dashed amber) ax.axhline(kc[ks[0]][0], color="#999", lw=0.8, ls=":") ax.axhline(kc[ks[0]][1], color="#999", lw=0.8, ls=":") + ax.axhline(0.457, color="#b7791f", lw=1.0, ls="--", alpha=0.7) ax.set_xticks(list(xs)) - ax.set_xticklabels([str(k) for k in ks], fontsize=8) - ax.set_title(title, fontsize=9) - ax.set_xlabel("k", fontsize=9) - ax.set_ylim(0, 1.02) -axes[0].set_ylabel("pass@1 (easy pastel / hard dark)") -axes[0].legend(fontsize=8, frameon=False, loc="center right") -fig.suptitle("Depth curves by regime — same frozen band, data, and 250-item " - "eval; dotted lines = k=0 base levels", fontsize=11, y=1.04) + ax.set_xticklabels([str(k) for k in ks], fontsize=7.5) + ax.set_xlabel("k", fontsize=8) +for i in (0, 5): + axf[i].set_ylabel("pass@1", fontsize=9) + +from matplotlib.lines import Line2D +axf[0].legend(handles=[ + Line2D([], [], color="#555", alpha=0.32, marker="o", lw=2.2, label="easy"), + Line2D([], [], color="#555", marker="s", lw=2.2, label="hard"), + Line2D([], [], color="#b7791f", ls="--", lw=1.0, + label="distill hard (45.7%, no loop)"), + Line2D([], [], color="#999", ls=":", lw=0.8, label="k=0 base levels"), +], fontsize=7, frameon=False, loc="center right") + +fig.suptitle("The design space, measured — depth curves for every regime " + "(same frozen band, data, 250-item eval; ★ = recommended recipe; " + "amber dashes = what distillation reaches with no loop at all)", + fontsize=11, y=1.0) fig.tight_layout() fig.savefig(OUT / "fig_kcurves.png", dpi=140, facecolor="white", bbox_inches="tight")