CPU endgame: stats pass (Wilson/McNemar/pooled hard), final figures, PAPER.md rewrite around amortizable-content thesis

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

results-loop/stats_final.json

@@ -0,0 +1,369 @@
+{
+ "MBPP loop s0 k=0 (base)": {
+  "overall": {
+   "acc": 0.518,
+   "n": 500,
+   "ci": [
+    0.474,
+    0.561
+   ]
+  },
+  "hard": {
+   "acc": 0.05454545454545454,
+   "n": 55,
+   "ci": [
+    0.019,
+    0.149
+   ]
+  }
+ },
+ "MBPP loop s0 k=2": {
+  "overall": {
+   "acc": 0.53,
+   "n": 500,
+   "ci": [
+    0.486,
+    0.573
+   ]
+  },
+  "hard": {
+   "acc": 0.3090909090909091,
+   "n": 55,
+   "ci": [
+    0.203,
+    0.44
+   ]
+  }
+ },
+ "MBPP loop s0 k=4": {
+  "overall": {
+   "acc": 0.536,
+   "n": 500,
+   "ci": [
+    0.492,
+    0.579
+   ]
+  },
+  "hard": {
+   "acc": 0.43636363636363634,
+   "n": 55,
+   "ci": [
+    0.314,
+    0.567
+   ]
+  }
+ },
+ "MBPP distill s1 k=1 (FF)": {
+  "overall": {
+   "acc": 0.544,
+   "n": 500,
+   "ci": [
+    0.5,
+    0.587
+   ]
+  },
+  "hard": {
+   "acc": 0.41818181818181815,
+   "n": 55,
+   "ci": [
+    0.297,
+    0.55
+   ]
+  }
+ },
+ "MBPP pause16 k=1": {
+  "overall": {
+   "acc": 0.552,
+   "n": 500,
+   "ci": [
+    0.508,
+    0.595
+   ]
+  },
+  "hard": {
+   "acc": 0.36363636363636365,
+   "n": 55,
+   "ci": [
+    0.249,
+    0.496
+   ]
+  }
+ },
+ "MBPP stack-train k=4": {
+  "overall": {
+   "acc": 0.524,
+   "n": 500,
+   "ci": [
+    0.48,
+    0.567
+   ]
+  },
+  "hard": {
+   "acc": 0.34545454545454546,
+   "n": 55,
+   "ci": [
+    0.234,
+    0.477
+   ]
+  }
+ },
+ "MBPP distill-in-loopmode k=2": {
+  "overall": {
+   "acc": 0.458,
+   "n": 500,
+   "ci": [
+    0.415,
+    0.502
+   ]
+  },
+  "hard": {
+   "acc": 0.2,
+   "n": 55,
+   "ci": [
+    0.116,
+    0.324
+   ]
+  }
+ },
+ "Rust transfer k=0": {
+  "overall": {
+   "acc": 0.5909090909090909,
+   "n": 154,
+   "ci": [
+    0.512,
+    0.665
+   ]
+  },
+  "hard": {
+   "acc": 0.08,
+   "n": 25,
+   "ci": [
+    0.022,
+    0.25
+   ]
+  }
+ },
+ "Rust transfer k=4": {
+  "overall": {
+   "acc": 0.551948051948052,
+   "n": 154,
+   "ci": [
+    0.473,
+    0.628
+   ]
+  },
+  "hard": {
+   "acc": 0.24,
+   "n": 25,
+   "ci": [
+    0.115,
+    0.434
+   ]
+  }
+ },
+ "HumanEval loop k=4": {
+  "overall": {
+   "acc": 0.6646341463414634,
+   "n": 164,
+   "ci": [
+    0.589,
+    0.732
+   ]
+  },
+  "hard": {
+   "acc": 0.3157894736842105,
+   "n": 38,
+   "ci": [
+    0.191,
+    0.475
+   ]
+  }
+ },
+ "HumanEval distill k=1": {
+  "overall": {
+   "acc": 0.6463414634146342,
+   "n": 164,
+   "ci": [
+    0.571,
+    0.715
+   ]
+  },
+  "hard": {
+   "acc": 0.23684210526315788,
+   "n": 38,
+   "ci": [
+    0.13,
+    0.392
+   ]
+  }
+ },
+ "MBPP best-of-3 (compute-matched)": {
+  "overall": {
+   "acc": 0.572,
+   "n": 500,
+   "ci": [
+    0.528,
+    0.615
+   ]
+  },
+  "hard": {
+   "acc": 0.32727272727272727,
+   "n": 55,
+   "ci": [
+    0.218,
+    0.459
+   ]
+  }
+ },
+ "MBPP budget-CoT-50": {
+  "overall": {
+   "acc": 0.538,
+   "n": 500,
+   "ci": [
+    0.494,
+    0.581
+   ]
+  },
+  "hard": {
+   "acc": 0.4,
+   "n": 55,
+   "ci": [
+    0.281,
+    0.532
+   ]
+  }
+ },
+ "distill_seed_spread": {
+  "hard": [
+   0.4909090909090909,
+   0.41818181818181815,
+   0.5454545454545454,
+   0.43636363636363634,
+   0.45454545454545453,
+   0.4727272727272727,
+   0.43636363636363634,
+   0.4
+  ],
+  "overall": [
+   0.55,
+   0.544,
+   0.562,
+   0.546,
+   0.566,
+   0.562,
+   0.564,
+   0.55
+  ]
+ },
+ "loop_seed_spread_hard": [
+  0.43636363636363634,
+  0.41818181818181815,
+  0.3090909090909091,
+  0.32727272727272727,
+  0.38181818181818183
+ ],
+ "mcnemar": {
+  "loop k=4 vs k=0, overall": {
+   "n": 500,
+   "a_only": 30,
+   "b_only": 39,
+   "p": 0.33555761823401514
+  },
+  "loop k=4 vs k=0, hard": {
+   "n": 55,
+   "a_only": 1,
+   "b_only": 22,
+   "p": 5.7220458984375e-06
+  },
+  "distill k=1 vs loop k=4, overall": {
+   "n": 500,
+   "a_only": 33,
+   "b_only": 37,
+   "p": 0.7202027723528613
+  },
+  "distill k=1 vs loop k=4, hard": {
+   "n": 55,
+   "a_only": 10,
+   "b_only": 9,
+   "p": 1.0
+  },
+  "stack-train k=4 vs distill k=1, hard": {
+   "n": 55,
+   "a_only": 10,
+   "b_only": 6,
+   "p": 0.454498291015625
+  },
+  "HumanEval loop k=4 vs k=0, overall": {
+   "n": 164,
+   "a_only": 5,
+   "b_only": 18,
+   "p": 0.010622024536132812
+  },
+  "HumanEval distill k=1 vs k=0, overall": {
+   "n": 164,
+   "a_only": 7,
+   "b_only": 17,
+   "p": 0.06391465663909912
+  },
+  "Rust loop k=4 vs k=0, overall": {
+   "n": 154,
+   "a_only": 11,
+   "b_only": 5,
+   "p": 0.210113525390625
+  },
+  "Rust loop k=4 vs k=0, hard": {
+   "n": 25,
+   "a_only": 0,
+   "b_only": 4,
+   "p": 0.125
+  }
+ },
+ "pooled_hard_loop": {
+  "base": 5,
+  "arm": 42,
+  "n": 118,
+  "mcnemar": {
+   "n": 118,
+   "a_only": 1,
+   "b_only": 38,
+   "p": 1.4551915228366852e-10
+  }
+ },
+ "pooled_hard_distill": {
+  "base": 3,
+  "arm": 32,
+  "n": 93,
+  "mcnemar": {
+   "n": 93,
+   "a_only": 1,
+   "b_only": 30,
+   "p": 2.9802322387695312e-08
+  }
+ },
+ "consensus_hard": {
+  "MBPP loop s0 k=4": {
+   "acc": 0.4230769230769231,
+   "n": 52,
+   "ci": [
+    0.299,
+    0.558
+   ]
+  },
+  "MBPP distill s1 k=1 (FF)": {
+   "acc": 0.40384615384615385,
+   "n": 52,
+   "ci": [
+    0.282,
+    0.539
+   ]
+  },
+  "MBPP stack-train k=4": {
+   "acc": 0.34615384615384615,
+   "n": 52,
+   "ci": [
+    0.232,
+    0.482
+   ]
+  }
+ }
+}