nils/jspace
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

n=500 controls: paired net-effect significant (loop vs untrained merge 17-4, p=0.007); ladder and net accounting finalized

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nils
2026-07-14 20:08:57 +02:00
co-authored by Claude Fable 5
parent 1d4cffd6ad
commit 0fd93cb328
8 changed files with 10160 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/stats_final.py
+11
View File
@@ -85,6 +85,10 @@ def main():
lines += ["## Headline numbers (Wilson 95% CIs)", ""]
ARMS = [
# (label, file, k, hard-subset)
("MBPP untrained merge k=4 (n=500 rerun)",
"eval_code_untrained_full.json", 4, mbpp_hard),
("MBPP trained FF k=1 (n=500 rerun)",
"eval_code_code_ff_full.json", 1, mbpp_hard),
("MBPP loop s0 k=0 (base)", "eval_code_code_s0_full.json", 0, mbpp_hard),
("MBPP loop s0 k=2", "eval_code_code_s0_full.json", 2, mbpp_hard),
("MBPP loop s0 k=4", "eval_code_code_s0_full.json", 4, mbpp_hard),
@@ -173,9 +177,16 @@ def main():
dist1, _ = arm_maps["MBPP distill s1 k=1 (FF)"]
stack4, _ = arm_maps["MBPP stack-train k=4"]
untr4, _ = arm_maps["MBPP untrained merge k=4 (n=500 rerun)"]
ff1, _ = arm_maps["MBPP trained FF k=1 (n=500 rerun)"]
TESTS = [
("loop k=4 vs k=0, overall", loop0, loop4, None),
("loop k=4 vs k=0, hard", loop0, loop4, mbpp_hard),
("loop k=4 vs UNTRAINED merge k=4, hard (net effect)",
untr4, loop4, mbpp_hard),
("loop k=4 vs UNTRAINED merge k=4, overall", untr4, loop4, None),
("trained FF vs UNTRAINED merge, hard", untr4, ff1, mbpp_hard),
("distill k=1 vs UNTRAINED merge k=4, hard", untr4, dist1, mbpp_hard),
("distill k=1 vs loop k=4, overall", loop4, dist1, None),
("distill k=1 vs loop k=4, hard", loop4, dist1, mbpp_hard),
("stack-train k=4 vs distill k=1, hard", dist1, stack4, mbpp_hard),