diff --git a/results-loop/PROTOCOL_UNIFIED.md b/results-loop/PROTOCOL_UNIFIED.md index da0f60e..ce3bd26 100644 --- a/results-loop/PROTOCOL_UNIFIED.md +++ b/results-loop/PROTOCOL_UNIFIED.md @@ -420,3 +420,22 @@ E1c amendment (pre-registered before running, same session): pre-loop halt consult on s_0 enabling k*=0; targets easy->0, hard->4; threshold 0.5 unchanged (calibration deferred unless E1c misses). Predictions: easy >= 95%, hard >= 35.7%, r >= 0.4, overall >= 51.2 at E[k] <= 1.5. + +--- Outcome, item 19 / E1c (scored 2026-07-16 ~01:50, Spark re-run): +prediction (b) CONFIRMED — easy 95.9% with mean k*=0.11 (the k=0 routing +fix worked perfectly for easy items); (a) FAILED (r=0.220, unchanged); +(c) FAILED HARD — hard 21.4% at mean k*=1.5: with the pre-loop halt +consult, weakly-discriminated hard items now exit at k*=0/1 where before +they got >=1 iteration; the recall problem became more expensive, not +better. (d) FAILED (overall 50.0 at E[k]=0.74). Net Pareto: E1c = +(50.0 overall, 95.9 easy, 21.4 hard, 0.74 mean iters — 82% compute +saved); E0 probe-gate = (52.0, 97.5, 28.6, ~2.2) still dominates on +accuracy. The learned-head line has ONE identified un-tried knob: +deploy-threshold calibration on val for hard-recall (the 0.5 threshold +is arbitrary; raising it routes more items deep, trading easy tax for +hard recall — a tunable curve E0 cannot offer). E1 arc summary for +PLAN_SELFPACED: gating machinery works end-to-end, easy-item protection +and compute savings are demonstrated and cheap; difficulty-selective +DEPTH allocation remains unsolved at 3K-param-head scale — binding +constraint is classifier quality on the k=0/s0 state, exactly where E0 +started. Next per plan: threshold sweep (cheap) before any E2. diff --git a/results-loop/eval_code_gatehead_c_e300.json b/results-loop/eval_code_gatehead_c_e300.json new file mode 100644 index 0000000..7d3c4d3 --- /dev/null +++ b/results-loop/eval_code_gatehead_c_e300.json @@ -0,0 +1,1519 @@ +{ + "tag": "gatehead_c_e300", + "acc": 0.5, + "by_label": { + "easy": 0.9590163934426229, + "hard": 0.21428571428571427, + "drop": 0.02 + }, + "mean_kstar": { + "easy": 0.11475409836065574, + "hard": 1.5, + "drop": 1.29 + }, + "corr_hard": 0.22035130858421326, + "kstars": [ + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4, + 0, + 0, + 0, + 0, + 4, + 0, + 1, + 2, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 3, + 0, + 0, + 1, + 0, + 0, + 0, + 3, + 0, + 0, + 0, + 0, + 2, + 3, + 3, + 0, + 0, + 0, + 0, + 0, + 4, + 1, + 0, + 0, + 4, + 2, + 0, + 3, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 1, + 2, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 2, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 3, + 0, + 0, + 3, + 0, + 0, + 0, + 0, + 4, + 2, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 3, + 3, + 2, + 0, + 1, + 1, + 1, + 0, + 4, + 0, + 0, + 0, + 0, + 2, + 0, + 1, + 1, + 2, + 0, + 0, + 4, + 4, + 0, + 0, + 0, + 0, + 2, + 2, + 4, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4, + 0, + 1, + 0, + 0, + 0, + 2, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 1, + 3, + 0, + 0, + 1, + 2, + 0, + 1, + 0, + 1, + 2, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 0, + 4, + 1, + 3, + 0, + 0, + 0, + 0, + 0, + 3, + 2, + 0, + 1, + 0, + 2, + 0, + 0, + 2, + 0, + 1, + 0, + 0, + 2, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 4, + 1, + 0, + 4, + 0, + 0, + 0, + 2, + 0, + 4, + 0, + 3, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 4 + ], + "per_item": [ + { + "task_id": 11, + "ok": true, + "kstar": 0 + }, + { + "task_id": 12, + "ok": false, + "kstar": 0 + }, + { + "task_id": 13, + "ok": false, + "kstar": 0 + }, + { + "task_id": 14, + "ok": true, + "kstar": 0 + }, + { + "task_id": 15, + "ok": false, + "kstar": 0 + }, + { + "task_id": 16, + "ok": true, + "kstar": 1 + }, + { + "task_id": 17, + "ok": true, + "kstar": 0 + }, + { + "task_id": 18, + "ok": true, + "kstar": 0 + }, + { + "task_id": 19, + "ok": true, + "kstar": 0 + }, + { + "task_id": 20, + "ok": false, + "kstar": 2 + }, + { + "task_id": 21, + "ok": false, + "kstar": 0 + }, + { + "task_id": 22, + "ok": true, + "kstar": 0 + }, + { + "task_id": 23, + "ok": true, + "kstar": 0 + }, + { + "task_id": 24, + "ok": false, + "kstar": 0 + }, + { + "task_id": 25, + "ok": true, + "kstar": 0 + }, + { + "task_id": 26, + "ok": false, + "kstar": 0 + }, + { + "task_id": 27, + "ok": true, + "kstar": 0 + }, + { + "task_id": 28, + "ok": true, + "kstar": 0 + }, + { + "task_id": 29, + "ok": true, + "kstar": 0 + }, + { + "task_id": 30, + "ok": true, + "kstar": 0 + }, + { + "task_id": 31, + "ok": false, + "kstar": 4 + }, + { + "task_id": 32, + "ok": true, + "kstar": 0 + }, + { + "task_id": 33, + "ok": false, + "kstar": 0 + }, + { + "task_id": 34, + "ok": false, + "kstar": 0 + }, + { + "task_id": 35, + "ok": false, + "kstar": 0 + }, + { + "task_id": 36, + "ok": false, + "kstar": 4 + }, + { + "task_id": 37, + "ok": true, + "kstar": 0 + }, + { + "task_id": 38, + "ok": false, + "kstar": 1 + }, + { + "task_id": 39, + "ok": false, + "kstar": 2 + }, + { + "task_id": 40, + "ok": true, + "kstar": 0 + }, + { + "task_id": 41, + "ok": true, + "kstar": 0 + }, + { + "task_id": 42, + "ok": false, + "kstar": 0 + }, + { + "task_id": 43, + "ok": false, + "kstar": 1 + }, + { + "task_id": 44, + "ok": true, + "kstar": 0 + }, + { + "task_id": 45, + "ok": true, + "kstar": 0 + }, + { + "task_id": 46, + "ok": true, + "kstar": 0 + }, + { + "task_id": 47, + "ok": false, + "kstar": 3 + }, + { + "task_id": 48, + "ok": false, + "kstar": 0 + }, + { + "task_id": 49, + "ok": true, + "kstar": 0 + }, + { + "task_id": 50, + "ok": false, + "kstar": 1 + }, + { + "task_id": 51, + "ok": true, + "kstar": 0 + }, + { + "task_id": 52, + "ok": true, + "kstar": 0 + }, + { + "task_id": 53, + "ok": true, + "kstar": 0 + }, + { + "task_id": 54, + "ok": true, + "kstar": 3 + }, + { + "task_id": 55, + "ok": true, + "kstar": 0 + }, + { + "task_id": 56, + "ok": true, + "kstar": 0 + }, + { + "task_id": 57, + "ok": true, + "kstar": 0 + }, + { + "task_id": 58, + "ok": true, + "kstar": 0 + }, + { + "task_id": 59, + "ok": true, + "kstar": 2 + }, + { + "task_id": 60, + "ok": false, + "kstar": 3 + }, + { + "task_id": 61, + "ok": false, + "kstar": 3 + }, + { + "task_id": 62, + "ok": true, + "kstar": 0 + }, + { + "task_id": 63, + "ok": true, + "kstar": 0 + }, + { + "task_id": 64, + "ok": true, + "kstar": 0 + }, + { + "task_id": 65, + "ok": true, + "kstar": 0 + }, + { + "task_id": 66, + "ok": true, + "kstar": 0 + }, + { + "task_id": 67, + "ok": false, + "kstar": 4 + }, + { + "task_id": 68, + "ok": false, + "kstar": 1 + }, + { + "task_id": 69, + "ok": true, + "kstar": 0 + }, + { + "task_id": 70, + "ok": true, + "kstar": 0 + }, + { + "task_id": 71, + "ok": false, + "kstar": 4 + }, + { + "task_id": 72, + "ok": false, + "kstar": 2 + }, + { + "task_id": 73, + "ok": true, + "kstar": 0 + }, + { + "task_id": 74, + "ok": false, + "kstar": 3 + }, + { + "task_id": 75, + "ok": true, + "kstar": 0 + }, + { + "task_id": 76, + "ok": false, + "kstar": 1 + }, + { + "task_id": 77, + "ok": false, + "kstar": 1 + }, + { + "task_id": 78, + "ok": true, + "kstar": 1 + }, + { + "task_id": 79, + "ok": true, + "kstar": 0 + }, + { + "task_id": 80, + "ok": true, + "kstar": 0 + }, + { + "task_id": 81, + "ok": false, + "kstar": 0 + }, + { + "task_id": 82, + "ok": true, + "kstar": 0 + }, + { + "task_id": 83, + "ok": false, + "kstar": 1 + }, + { + "task_id": 84, + "ok": false, + "kstar": 2 + }, + { + "task_id": 85, + "ok": true, + "kstar": 0 + }, + { + "task_id": 86, + "ok": true, + "kstar": 1 + }, + { + "task_id": 87, + "ok": false, + "kstar": 0 + }, + { + "task_id": 88, + "ok": true, + "kstar": 0 + }, + { + "task_id": 89, + "ok": false, + "kstar": 0 + }, + { + "task_id": 90, + "ok": true, + "kstar": 0 + }, + { + "task_id": 91, + "ok": true, + "kstar": 0 + }, + { + "task_id": 92, + "ok": false, + "kstar": 2 + }, + { + "task_id": 93, + "ok": true, + "kstar": 0 + }, + { + "task_id": 94, + "ok": true, + "kstar": 0 + }, + { + "task_id": 95, + "ok": true, + "kstar": 1 + }, + { + "task_id": 96, + "ok": true, + "kstar": 0 + }, + { + "task_id": 97, + "ok": true, + "kstar": 0 + }, + { + "task_id": 98, + "ok": true, + "kstar": 0 + }, + { + "task_id": 99, + "ok": true, + "kstar": 0 + }, + { + "task_id": 100, + "ok": false, + "kstar": 3 + }, + { + "task_id": 101, + "ok": false, + "kstar": 0 + }, + { + "task_id": 102, + "ok": true, + "kstar": 0 + }, + { + "task_id": 103, + "ok": false, + "kstar": 3 + }, + { + "task_id": 104, + "ok": true, + "kstar": 0 + }, + { + "task_id": 105, + "ok": true, + "kstar": 0 + }, + { + "task_id": 106, + "ok": false, + "kstar": 0 + }, + { + "task_id": 107, + "ok": false, + "kstar": 0 + }, + { + "task_id": 108, + "ok": false, + "kstar": 4 + }, + { + "task_id": 109, + "ok": false, + "kstar": 2 + }, + { + "task_id": 110, + "ok": false, + "kstar": 2 + }, + { + "task_id": 111, + "ok": false, + "kstar": 0 + }, + { + "task_id": 112, + "ok": false, + "kstar": 0 + }, + { + "task_id": 113, + "ok": true, + "kstar": 0 + }, + { + "task_id": 114, + "ok": false, + "kstar": 0 + }, + { + "task_id": 115, + "ok": true, + "kstar": 0 + }, + { + "task_id": 116, + "ok": true, + "kstar": 0 + }, + { + "task_id": 117, + "ok": true, + "kstar": 0 + }, + { + "task_id": 118, + "ok": false, + "kstar": 0 + }, + { + "task_id": 119, + "ok": false, + "kstar": 1 + }, + { + "task_id": 120, + "ok": false, + "kstar": 0 + }, + { + "task_id": 121, + "ok": false, + "kstar": 3 + }, + { + "task_id": 122, + "ok": false, + "kstar": 3 + }, + { + "task_id": 123, + "ok": false, + "kstar": 2 + }, + { + "task_id": 124, + "ok": false, + "kstar": 0 + }, + { + "task_id": 125, + "ok": false, + "kstar": 1 + }, + { + "task_id": 126, + "ok": false, + "kstar": 1 + }, + { + "task_id": 127, + "ok": true, + "kstar": 1 + }, + { + "task_id": 128, + "ok": true, + "kstar": 0 + }, + { + "task_id": 129, + "ok": false, + "kstar": 4 + }, + { + "task_id": 130, + "ok": false, + "kstar": 0 + }, + { + "task_id": 131, + "ok": true, + "kstar": 0 + }, + { + "task_id": 132, + "ok": true, + "kstar": 0 + }, + { + "task_id": 133, + "ok": true, + "kstar": 0 + }, + { + "task_id": 134, + "ok": false, + "kstar": 2 + }, + { + "task_id": 135, + "ok": true, + "kstar": 0 + }, + { + "task_id": 136, + "ok": false, + "kstar": 1 + }, + { + "task_id": 137, + "ok": false, + "kstar": 1 + }, + { + "task_id": 138, + "ok": false, + "kstar": 2 + }, + { + "task_id": 139, + "ok": false, + "kstar": 0 + }, + { + "task_id": 140, + "ok": false, + "kstar": 0 + }, + { + "task_id": 141, + "ok": false, + "kstar": 4 + }, + { + "task_id": 142, + "ok": false, + "kstar": 4 + }, + { + "task_id": 143, + "ok": false, + "kstar": 0 + }, + { + "task_id": 144, + "ok": true, + "kstar": 0 + }, + { + "task_id": 145, + "ok": true, + "kstar": 0 + }, + { + "task_id": 146, + "ok": false, + "kstar": 0 + }, + { + "task_id": 147, + "ok": false, + "kstar": 2 + }, + { + "task_id": 148, + "ok": false, + "kstar": 2 + }, + { + "task_id": 149, + "ok": true, + "kstar": 4 + }, + { + "task_id": 150, + "ok": false, + "kstar": 3 + }, + { + "task_id": 151, + "ok": true, + "kstar": 0 + }, + { + "task_id": 152, + "ok": false, + "kstar": 0 + }, + { + "task_id": 153, + "ok": true, + "kstar": 0 + }, + { + "task_id": 154, + "ok": true, + "kstar": 0 + }, + { + "task_id": 155, + "ok": false, + "kstar": 0 + }, + { + "task_id": 156, + "ok": true, + "kstar": 0 + }, + { + "task_id": 157, + "ok": true, + "kstar": 0 + }, + { + "task_id": 158, + "ok": false, + "kstar": 4 + }, + { + "task_id": 159, + "ok": false, + "kstar": 0 + }, + { + "task_id": 160, + "ok": false, + "kstar": 1 + }, + { + "task_id": 161, + "ok": true, + "kstar": 0 + }, + { + "task_id": 162, + "ok": true, + "kstar": 0 + }, + { + "task_id": 163, + "ok": true, + "kstar": 0 + }, + { + "task_id": 164, + "ok": false, + "kstar": 2 + }, + { + "task_id": 165, + "ok": false, + "kstar": 0 + }, + { + "task_id": 166, + "ok": false, + "kstar": 1 + }, + { + "task_id": 167, + "ok": true, + "kstar": 0 + }, + { + "task_id": 168, + "ok": true, + "kstar": 0 + }, + { + "task_id": 169, + "ok": false, + "kstar": 0 + }, + { + "task_id": 170, + "ok": false, + "kstar": 0 + }, + { + "task_id": 171, + "ok": true, + "kstar": 0 + }, + { + "task_id": 172, + "ok": true, + "kstar": 0 + }, + { + "task_id": 173, + "ok": true, + "kstar": 0 + }, + { + "task_id": 174, + "ok": true, + "kstar": 0 + }, + { + "task_id": 175, + "ok": true, + "kstar": 0 + }, + { + "task_id": 176, + "ok": true, + "kstar": 0 + }, + { + "task_id": 177, + "ok": false, + "kstar": 3 + }, + { + "task_id": 178, + "ok": true, + "kstar": 1 + }, + { + "task_id": 179, + "ok": true, + "kstar": 3 + }, + { + "task_id": 180, + "ok": false, + "kstar": 0 + }, + { + "task_id": 181, + "ok": false, + "kstar": 0 + }, + { + "task_id": 182, + "ok": false, + "kstar": 1 + }, + { + "task_id": 183, + "ok": false, + "kstar": 2 + }, + { + "task_id": 184, + "ok": false, + "kstar": 0 + }, + { + "task_id": 185, + "ok": false, + "kstar": 1 + }, + { + "task_id": 186, + "ok": true, + "kstar": 0 + }, + { + "task_id": 187, + "ok": false, + "kstar": 1 + }, + { + "task_id": 188, + "ok": false, + "kstar": 2 + }, + { + "task_id": 189, + "ok": false, + "kstar": 1 + }, + { + "task_id": 190, + "ok": false, + "kstar": 0 + }, + { + "task_id": 191, + "ok": true, + "kstar": 0 + }, + { + "task_id": 192, + "ok": true, + "kstar": 0 + }, + { + "task_id": 193, + "ok": false, + "kstar": 0 + }, + { + "task_id": 194, + "ok": true, + "kstar": 0 + }, + { + "task_id": 195, + "ok": false, + "kstar": 0 + }, + { + "task_id": 196, + "ok": true, + "kstar": 0 + }, + { + "task_id": 197, + "ok": true, + "kstar": 0 + }, + { + "task_id": 198, + "ok": false, + "kstar": 4 + }, + { + "task_id": 199, + "ok": true, + "kstar": 0 + }, + { + "task_id": 200, + "ok": true, + "kstar": 0 + }, + { + "task_id": 201, + "ok": true, + "kstar": 0 + }, + { + "task_id": 202, + "ok": true, + "kstar": 0 + }, + { + "task_id": 203, + "ok": true, + "kstar": 1 + }, + { + "task_id": 204, + "ok": true, + "kstar": 0 + }, + { + "task_id": 205, + "ok": false, + "kstar": 1 + }, + { + "task_id": 206, + "ok": false, + "kstar": 0 + }, + { + "task_id": 207, + "ok": false, + "kstar": 4 + }, + { + "task_id": 208, + "ok": false, + "kstar": 1 + }, + { + "task_id": 209, + "ok": false, + "kstar": 3 + }, + { + "task_id": 210, + "ok": true, + "kstar": 0 + }, + { + "task_id": 211, + "ok": false, + "kstar": 0 + }, + { + "task_id": 212, + "ok": false, + "kstar": 0 + }, + { + "task_id": 213, + "ok": false, + "kstar": 0 + }, + { + "task_id": 214, + "ok": true, + "kstar": 0 + }, + { + "task_id": 215, + "ok": false, + "kstar": 3 + }, + { + "task_id": 216, + "ok": false, + "kstar": 2 + }, + { + "task_id": 217, + "ok": true, + "kstar": 0 + }, + { + "task_id": 218, + "ok": false, + "kstar": 1 + }, + { + "task_id": 219, + "ok": false, + "kstar": 0 + }, + { + "task_id": 220, + "ok": false, + "kstar": 2 + }, + { + "task_id": 221, + "ok": true, + "kstar": 0 + }, + { + "task_id": 222, + "ok": true, + "kstar": 0 + }, + { + "task_id": 223, + "ok": true, + "kstar": 2 + }, + { + "task_id": 224, + "ok": true, + "kstar": 0 + }, + { + "task_id": 225, + "ok": false, + "kstar": 1 + }, + { + "task_id": 226, + "ok": true, + "kstar": 0 + }, + { + "task_id": 227, + "ok": true, + "kstar": 0 + }, + { + "task_id": 228, + "ok": false, + "kstar": 2 + }, + { + "task_id": 229, + "ok": false, + "kstar": 0 + }, + { + "task_id": 230, + "ok": true, + "kstar": 0 + }, + { + "task_id": 231, + "ok": false, + "kstar": 1 + }, + { + "task_id": 232, + "ok": true, + "kstar": 0 + }, + { + "task_id": 233, + "ok": false, + "kstar": 0 + }, + { + "task_id": 234, + "ok": true, + "kstar": 0 + }, + { + "task_id": 235, + "ok": false, + "kstar": 0 + }, + { + "task_id": 236, + "ok": false, + "kstar": 4 + }, + { + "task_id": 237, + "ok": false, + "kstar": 1 + }, + { + "task_id": 238, + "ok": true, + "kstar": 0 + }, + { + "task_id": 239, + "ok": false, + "kstar": 4 + }, + { + "task_id": 240, + "ok": true, + "kstar": 0 + }, + { + "task_id": 241, + "ok": false, + "kstar": 0 + }, + { + "task_id": 242, + "ok": true, + "kstar": 0 + }, + { + "task_id": 243, + "ok": false, + "kstar": 2 + }, + { + "task_id": 244, + "ok": true, + "kstar": 0 + }, + { + "task_id": 245, + "ok": false, + "kstar": 4 + }, + { + "task_id": 246, + "ok": true, + "kstar": 0 + }, + { + "task_id": 247, + "ok": true, + "kstar": 3 + }, + { + "task_id": 248, + "ok": false, + "kstar": 0 + }, + { + "task_id": 249, + "ok": false, + "kstar": 1 + }, + { + "task_id": 250, + "ok": true, + "kstar": 0 + }, + { + "task_id": 251, + "ok": true, + "kstar": 0 + }, + { + "task_id": 252, + "ok": true, + "kstar": 0 + }, + { + "task_id": 253, + "ok": true, + "kstar": 0 + }, + { + "task_id": 254, + "ok": true, + "kstar": 0 + }, + { + "task_id": 255, + "ok": false, + "kstar": 1 + }, + { + "task_id": 256, + "ok": false, + "kstar": 0 + }, + { + "task_id": 257, + "ok": true, + "kstar": 0 + }, + { + "task_id": 258, + "ok": false, + "kstar": 0 + }, + { + "task_id": 259, + "ok": false, + "kstar": 1 + }, + { + "task_id": 260, + "ok": false, + "kstar": 4 + } + ] +} \ No newline at end of file