From 3a0a6a96aea825f6bdadc10367041bb982eb5293 Mon Sep 17 00:00:00 2001 From: Nils Date: Wed, 15 Jul 2026 21:52:26 +0200 Subject: [PATCH] INDEX.md (task-oriented repo guide), REGIMES.json -> results/, phase diagram updated with full sweep (anchored family, seed means) Co-Authored-By: Claude Fable 5 --- INDEX.md | 83 ++++++++++++++++++++++++++ {results-loop => results}/REGIMES.json | 2 +- scripts/fig_phase.py | 36 ++++++----- 3 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 INDEX.md rename {results-loop => results}/REGIMES.json (91%) diff --git a/INDEX.md b/INDEX.md new file mode 100644 index 0000000..1a53346 --- /dev/null +++ b/INDEX.md @@ -0,0 +1,83 @@ +# Where to find what + +Two projects share this repo: the **J-lens reproduction** (does the 2026 +workspace paper replicate on gemma-4-E2B?) and the **workspace-looping +investigation** that grew out of it (retrofit recurrence onto the lens-found +band; what does it actually buy?). The second is the active one. + +## The claims and their evidence + +| you want | look in | +|---|---| +| Current claims, all numbers, figures | `PAPER.md` (source of truth; the .pdf snapshots lag it) | +| What worked / what failed / design rules / ops pitfalls | `LESSONS.md` — read before running anything on this hardware | +| Pre-registrations + scored outcomes (17 items, incl. refutations) | `results-loop/PROTOCOL_UNIFIED.md` — the methods backbone; every claim in PAPER.md §3 traces to an item here | +| Significance tests behind any claimed number | `results-loop/STATS.md` | +| Lab-notebook narrative of the looping investigation | `WORKSPACE_LOOPING.md` (superseded where it disagrees with PAPER.md) | +| Base-reproduction results (lens replication itself) | `RESULTS.md`, `README.md` | +| Per-model lens maps: workspace bands, KV-share boundaries, pinned revisions | `results/REGIMES.json` (canonical registry) + `results/jbar*.pt` (raw J̄) + `results/exp4*.log` (regime scans) | + +## Code (`scripts/`, `jlens/`) + +- `jlens/core.py` — the lens: model loading (`JLENS_MODEL` env), J̄ readouts. +- `scripts/loop_common.py` — everything band-looping: `BandLooper` + (capture/re-run machinery, KV-cache-safe), `generate_frozen_prompt` + (the ≥3.5× deploy path), and every adapter variant from the regime sweep + (`MergeAdapter` ★, `AdaptiveMergeAdapter`, `RecurrentAdapter`, + `ParcaeAdapter`, `NoisyMergeAdapter`, `TiedAlphaAdapter`, + `PerDepthAdapter`). Band via `JLENS_BAND` env (default E2B 14,30). +- Trainers: `train_merge_code.py` (MBPP; all regime flags live here), + `train_merge.py` (GSM, old full-position regime — historic), + `train_merge_unified.py` (multi-task, hardened protocol), + `train_merge_bw.py` (Blocksworld), `train_distill*.py` (plan distillation). +- Evals: `eval_loop_code.py` (MBPP pass@1 vs k; per-item logs; `--halt`), + `eval_loop.py` (GSM), `eval_bw.py`, `eval_humaneval.py`, `eval_rust.py`, + `eval_lcb.py`, `eval_mc_panel.py`, plus `prep_*.py` (STaR labeling). +- Figures: `fig_*.py` regenerate the canonical PNGs from the JSONs. +- Infra: `gpuq_*.sh` + `GPUQ.md` (bucket-backed GPU job queue), + `node_setup.sh` (vast.ai bootstrap; pins model revisions — see LESSONS #12), + `vast-ai-notes.md`. + +## Results directories — including the honest mess + +- `results-loop/` — **the looping project's data**: 84 `eval_*.json` + (tag suffixes: `_s`, `_rec16`/`_parcae16` recurrent arms, `_pd4` + per-depth, `_ta` tied-alpha, `_rk16` random-depth, `_ns` noise-s₀, + `_h2048` capacity, `code2gsm_*` transfer; `per_item` only in files from + Jul 14 onward), adapter checkpoints (`adapter_*.pt`, e400 = the + pre-committed eval checkpoint), canonical figures (`fig_kcurves.png` + design-space grid, `fig_phase.png` two-dials diagram, `fig_loop_vs_ff.png` + recurrence-vs-distill ladder, `fig_placement/transfer/scale.png`), + and `chain*.log` — autonomous-session logs, archaeology only. +- `results/` — lens reproduction outputs + the cross-model registry + (`REGIMES.json`). +- `results-band-*/` — one directory per entrance-placement arm of the + placement sweep (L2–L24 entrances); summarized in PAPER fig_placement; + kept for per-item audit. +- `results-12b/`, `results-loop-12b/` — 12B lens map and looping arms. +- `results-tap23/`, `results-tap34/`, `results-kvtest/`, `results-combo/`, + `results-panel/`, `results-distill-s7/` — single-question side arms + (exit-tap sweep, KV nulling check, combined arms, MC panel, distill seed). +- `results-node*/`, `results-node2-final/` — raw syncs from rented H100 + nodes (500-item eval campaign). +- `results-26b/` — **unclear provenance** (Jul 13; layer indices ≤26 mean + it is NOT the 26B MoE despite the name — possibly a misnamed early scan). + Trust nothing here without re-derivation. +- `paper-A/`, `paper-B/`, `paper-D/` — abandoned paper-outline variants + (one PLAN.md each); the live outline is PAPER.md itself. +- `related_work/` — the two anchor papers (McLeish 2511.07384, + Lys 2602.14759), the workspace paper, `relevant_to_us.md` notes, + `bibliography.bib`. + +## Conventions worth knowing + +- Every eval prints a `k=0` row first; it must equal the base model + bit-exactly (0.488 on MBPP-250) — the sanity anchor that has caught two + silent bugs (LESSONS #2, #12). +- Difficulty labels (`easy`/`hard`/`drop`) are STaR self-labels: + direct-pass / CoT-only-pass / unreachable. "hard" = plan-dependent. +- Checkpoints are pre-committed before evals (usually e400); post-hoc + checkpoint shopping is flagged as exploratory wherever it happened. +- GPU jobs go through the gpuq queue (`gpuq_submit.sh `), + never bare nohup on the Spark; jobs are killed by `pkill -f` self-matches + embarrassingly often (LESSONS #6). diff --git a/results-loop/REGIMES.json b/results/REGIMES.json similarity index 91% rename from results-loop/REGIMES.json rename to results/REGIMES.json index 02ac38c..14c7bf4 100644 --- a/results-loop/REGIMES.json +++ b/results/REGIMES.json @@ -1,5 +1,5 @@ { - "_doc": "Canonical per-model J-lens regime registry. Band = inclusive layer range whose lens readout shows workspace behavior (persist+content high, sensor/motor low; exp4_regimes.py). kv_share_start = first layer reusing KV from below (entrances at or above it are structurally null for band looping). Raw jbar tensors: results/ + bucket results-lens/. Revisions are the pinned model SHAs the scan ran on.", + "_doc": "Canonical per-model J-lens regime registry. Band = inclusive layer range whose lens readout shows workspace behavior (persist+content high, sensor/motor low; exp4_regimes.py). kv_share_start = first layer reusing KV from below (entrances at or above it are structurally null for band looping). Lives in results/ next to the raw jbar tensors and exp4 scan logs; bucket mirror: results-lens/. Revisions are the pinned model SHAs the scan ran on.", "google/gemma-4-E2B-it": { "revision": "9dbdf8a839e4e9e0eb56ed80cc8886661d3817cf", "num_layers": 35, diff --git a/scripts/fig_phase.py b/scripts/fig_phase.py index 56bc9c0..2c6dc20 100644 --- a/scripts/fig_phase.py +++ b/scripts/fig_phase.py @@ -16,11 +16,15 @@ OUT = Path(__file__).resolve().parent.parent / "results-loop" # name rho easy hard color # name rho(plot x) easy hard color dy PTS = [ - ("untrained α-merge\n(training-free)\nhard 14%", 0.40, 0.885, 0.143, "#8a8f98", -52), - ("trained merge\n(anchored B, curriculum)", 0.26, 0.885, 0.464, "#2b6cb0", -60), - ("per-depth merges\n(time-varying, anchored)\nhard 36% (depth-stranded)", 0.325, 0.893, 0.357, "#805ad5", 18), - ("Parcae rec\n(ρ<1 enforced, learned B)", 0.292, 0.713, 0.429, "#2f855a", -58), - ("unconstrained rec\n(learned A,B)", 4.5, 0.697, 0.393, "#c53030", 16), + ("untrained 14%", 0.22, 0.885, 0.143, "#8a8f98", -26), + ("merge ★ 46%", 0.26, 0.885, 0.464, "#2b6cb0", -40), + ("per-depth 36%", 0.30, 0.893, 0.357, "#805ad5", 26), + ("tied-α 43%", 0.35, 0.910, 0.429, "#0987a0", -32), + ("rand-k 36%", 0.41, 0.918, 0.357, "#3182ce", 22), + ("noise-s₀ 43%*", 0.48, 0.904, 0.429, "#b83280", -34), + ("4×MLP 54%†", 0.56, 0.902, 0.536, "#5f6b7a", 20), + ("Parcae rec 41%*\n(ρ<1 enforced, learned B)", 0.292, 0.722, 0.411, "#2f855a", -50), + ("unconstrained rec 39%\n(learned A,B)", 4.5, 0.697, 0.393, "#c53030", 16), ] # x jittered around true rho=0.3 for the three anchored arms (visibility) # untrained merge x offset for visibility (true rho = 0.30) @@ -38,25 +42,29 @@ ax.text(3.1, 0.615, "norm projection converts\nexplosion → stationary churn\n" for name, rho, easy, hard, c, dy in PTS: ax.scatter(rho, easy, s=90 + 900 * hard, color=c, alpha=0.85, zorder=3, edgecolor="white", linewidth=1.5) - lbl = name if ("stranded" in name or "14%" in name) else f"{name}\nhard {hard:.0%}" - dx = 62 if "14%" in name else 0 + lbl = name + dx = 0 ax.annotate(lbl, (rho, easy), textcoords="offset points", xytext=(dx, dy), - ha="center", fontsize=8.5, color=c) + ha="center", fontsize=7.5, color=c) -ax.annotate("", xy=(0.278, 0.735), xytext=(0.288, 0.862), +ax.annotate("", xy=(0.285, 0.740), xytext=(0.29, 0.868), arrowprops=dict(arrowstyle="->", color="#2f855a", lw=1.3)) -ax.text(0.46, 0.775, "learned B + no curriculum:\nfixed point leaves the\n" - "substrate manifold", fontsize=8, color="#2f855a", ha="left") +ax.text(0.55, 0.79, "learned free B:\nfixed point leaves the\n" + "substrate manifold\n(curriculum, s₀, ρ, capacity\nall causally cleared)", + fontsize=8, color="#2f855a", ha="left") +ax.text(0.34, 0.972, "the anchored family — B tied to the anchor (labels: hard-bucket at k=4; * seed mean, † single seed)", + fontsize=7.8, color="#2b6cb0", ha="center", style="italic") -ax.set_xlim(0.2, 12) +ax.set_xlim(0.17, 12) ax.set_ylim(0.58, 1.0) ax.set_xlabel("ρ(A) — spectral radius of the trained state map (dynamics dial)") ax.set_ylabel("substrate fidelity — easy-bucket pass@1 at k=4") ax.set_title("Frozen-band recurrence phase diagram: stability ≠ fidelity\n" - "(marker size ∝ hard-bucket gain; every regime buys the same " - "~40–46%, only one keeps the substrate)", fontsize=10.5) + "(marker size ∝ hard-bucket gain at k=4; every regime buys " + "36–54% within seed noise; only anchored B keeps the substrate)", + fontsize=10.5) ax.grid(True, color="#ececec", lw=0.7, which="both") ax.set_axisbelow(True) for s in ("top", "right"):