nils/jspace
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

n=500 controls: paired net-effect significant (loop vs untrained merge 17-4, p=0.007); ladder and net accounting finalized

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nils
2026-07-14 20:08:57 +02:00
co-authored by Claude Fable 5
parent 1d4cffd6ad
commit 0fd93cb328
8 changed files with 10160 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/figures_final.py
+4 -7
View File
@@ -100,13 +100,10 @@ def fig_ladder():
_, h0 = ev(OUT / "eval_code_code_s0_full.json", 0)
add("base (k=0, exact)", h0, NH, GRAY)
# untrained E2B control (250-item era, n_hard=28)
d = json.load(open(OUT / "eval_code_untrained.json"))
hu = max(v["by_label"]["hard"] for k, v in d["ks"].items() if k != "0")
add("untrained loop (best k)", hu, 28, GRAY)
d = json.load(open(OUT / "eval_code_ff.json"))
add("trained FF (no recurrence)", d["ks"]["1"]["by_label"]["hard"], 28,
ORANGE)
_, hu = ev(OUT / "eval_code_untrained_full.json", 4)
add("untrained loop (perturbation floor)", hu, NH, GRAY)
_, hf = ev(OUT / "eval_code_code_ff_full.json", 1)
add("trained FF (no recurrence)", hf, NH, ORANGE)
_, hp = ev(OUT / "eval_code_pause16.json", 1)
add("pause-16 registers (width)", hp, NH, ORANGE)
_, hl = ev(OUT / "eval_code_code_s0_full.json", 4)
scripts/stats_final.py
+11
View File
@@ -85,6 +85,10 @@ def main():
lines += ["## Headline numbers (Wilson 95% CIs)", ""]
ARMS = [
# (label, file, k, hard-subset)
("MBPP untrained merge k=4 (n=500 rerun)",
"eval_code_untrained_full.json", 4, mbpp_hard),
("MBPP trained FF k=1 (n=500 rerun)",
"eval_code_code_ff_full.json", 1, mbpp_hard),
("MBPP loop s0 k=0 (base)", "eval_code_code_s0_full.json", 0, mbpp_hard),
("MBPP loop s0 k=2", "eval_code_code_s0_full.json", 2, mbpp_hard),
("MBPP loop s0 k=4", "eval_code_code_s0_full.json", 4, mbpp_hard),
@@ -173,9 +177,16 @@ def main():
dist1, _ = arm_maps["MBPP distill s1 k=1 (FF)"]
stack4, _ = arm_maps["MBPP stack-train k=4"]
untr4, _ = arm_maps["MBPP untrained merge k=4 (n=500 rerun)"]
ff1, _ = arm_maps["MBPP trained FF k=1 (n=500 rerun)"]
TESTS = [
("loop k=4 vs k=0, overall", loop0, loop4, None),
("loop k=4 vs k=0, hard", loop0, loop4, mbpp_hard),
("loop k=4 vs UNTRAINED merge k=4, hard (net effect)",
untr4, loop4, mbpp_hard),
("loop k=4 vs UNTRAINED merge k=4, overall", untr4, loop4, None),
("trained FF vs UNTRAINED merge, hard", untr4, ff1, mbpp_hard),
("distill k=1 vs UNTRAINED merge k=4, hard", untr4, dist1, mbpp_hard),
("distill k=1 vs loop k=4, overall", loop4, dist1, None),
("distill k=1 vs loop k=4, hard", loop4, dist1, mbpp_hard),
("stack-train k=4 vs distill k=1, hard", dist1, stack4, mbpp_hard),