diff --git a/results-loop/PROTOCOL_UNIFIED.md b/results-loop/PROTOCOL_UNIFIED.md index bbe6e11..0e81d52 100644 --- a/results-loop/PROTOCOL_UNIFIED.md +++ b/results-loop/PROTOCOL_UNIFIED.md @@ -583,3 +583,20 @@ curriculum effect + the reach trade, not scratchpad-free parity. Next knobs if the line continues: longer per-rung retraining, finer deletion schedule (fractional rungs), or E2-N noise-hardening stacked on the rungs. + +23. **E2-L d=1 capacity-ceiling controls (pre-registered 2026-07-16 + ~15:55, before running; Nils: "run that one control").** Item 22's + break-at-d=1 has two untested confounds; one arm each, single-knob + changes from item 22's d=1 (both warm-start rung-A e400, seed 0): + arm A "x600" = 600 steps instead of 200 (3x training; tests the + brief-retrain objection — note d=1 val had plateaued at 0.36, so + prediction is NO recovery); arm B "pp30" = 30 pauses per deleted + step instead of 10 (3x latent bandwidth, same 200 steps; tests + whether inert-pause capacity, not training, binds). Eval n=256: + arm A cells 0:0, 2:12, 2:16 (e600); arm B cells 0:0, 2:32, 2:36 + (e200). Decision rule, stated in advance: item 22 d=1 best = 31.6; + a control within +-5 points confirms the ceiling on that axis; + >=+5 (>36.6) reopens the ladder on that knob (longer schedules or + wider pause-chains); if BOTH land within +-5, the d=1 ceiling is + confirmed structural and the ladder chapter closes as scored. + Job: scripts/jobs/zzz_n_rungb_ctrl.sh. diff --git a/scripts/jobs/zzz_n_rungb_ctrl.sh b/scripts/jobs/zzz_n_rungb_ctrl.sh new file mode 100644 index 0000000..239866a --- /dev/null +++ b/scripts/jobs/zzz_n_rungb_ctrl.sh @@ -0,0 +1,13 @@ +# gpuq-in: results-loop/star_data.json results-loop/gsm_cot_data.json results-loop/adapter_carrycot_e400.pt +# gpuq-out: results-loop/eval_gsm_carrycot_b1_x600.json results-loop/eval_gsm_carrycot_b1_pp30.json results-loop/train_carrycot_b1_x600_log.json results-loop/train_carrycot_b1_pp30_log.json results-loop/adapter_carrycot_b1_x600_*.pt results-loop/adapter_carrycot_b1_pp30_*.pt +git pull origin main -q 2>/dev/null +P=/home/nils/jspace/.venv/bin/python +export JLENS_MODEL=google/gemma-4-E2B-it LOOP_OUT=/home/nils/jspace/results-loop +cd /home/nils/jspace/scripts +WARM=$LOOP_OUT/adapter_carrycot_e400.pt +# arm A (undertrain control): d=1, 3x the training, same 10 pauses +$P train_carry_cot.py --drop-steps 1 --warm-start $WARM --steps 600 --lr 3e-4 --tag-suffix _x600 +$P eval_carry_cot.py --adapter $LOOP_OUT/adapter_carrycot_b1_x600_e600.pt --tag gsm_carrycot_b1_x600 --grid 0:0,2:12,2:16 --n 256 +# arm B (pause-bandwidth control): d=1, same 200 steps, 3x the pauses +$P train_carry_cot.py --drop-steps 1 --warm-start $WARM --steps 200 --lr 3e-4 --pause-per-step 30 --tag-suffix _pp30 +$P eval_carry_cot.py --adapter $LOOP_OUT/adapter_carrycot_b1_pp30_e200.pt --tag gsm_carrycot_b1_pp30 --grid 0:0,2:32,2:36 --n 256 diff --git a/scripts/train_carry_cot.py b/scripts/train_carry_cot.py index 6f1b7b5..9831cfb 100644 --- a/scripts/train_carry_cot.py +++ b/scripts/train_carry_cot.py @@ -51,12 +51,14 @@ ap.add_argument("--pause-per-step", type=int, default=10, ap.add_argument("--warm-start", default=None, metavar="ADAPTER_PT") ap.add_argument("--steps", type=int, default=STEPS) ap.add_argument("--lr", type=float, default=LR) +ap.add_argument("--tag-suffix", default="", + help="appended to TAG (distinguish control variants)") ARGS = ap.parse_args() STEPS, LR = ARGS.steps, ARGS.lr TAG = ("carrycot_ff" if ARGS.feedforward else "carrycot") + ( f"_b{ARGS.drop_steps}" if ARGS.drop_steps else "") + ( f"_ln{ARGS.lensnoise.replace(',', '_')}" if ARGS.lensnoise else "") + ( - f"_s{ARGS.seed}" if ARGS.seed else "") + f"_s{ARGS.seed}" if ARGS.seed else "") + ARGS.tag_suffix def drop_cot_steps(cot, d):