diff --git a/results-loop/PROTOCOL_UNIFIED.md b/results-loop/PROTOCOL_UNIFIED.md index de66385..d3dc8bc 100644 --- a/results-loop/PROTOCOL_UNIFIED.md +++ b/results-loop/PROTOCOL_UNIFIED.md @@ -454,3 +454,43 @@ started. Next per plan: threshold sweep (cheap) before any E2. overall >= 55% — if so, gate-quality headroom is large and further gate work is justified; if oracle < 53%, gating this merge is nearly saturated and the program pivots to E2 or closes. + +--- Outcome, item 20 (scored 2026-07-16 ~02:35): (b) CONFIRMED — clean +monotone threshold curve (hard 7->50%, easy 96.7->87.7%, E[k] 0.43->2.63 +across theta .3->.99). (a) FAILED — no theta reaches easy>=93 AND +hard>=32; at matched easy the E0 frozen probe dominates the entire +learned-head curve: the BCE-trained 3K head is strictly worse than the +class-balanced logistic probe it was meant to replace. (c) CONFIRMED, +emphatically: ORACLE gate = 59.6 overall / easy 100% / hard 64.3% at +E[k]=0.24. Key insight: hard items are DEPTH-DIVERSE — 18/28 solvable at +some k in {0,1,2,4} but no single k solves more than 13; a third of the +hard bucket lives in per-item depth selection. Program continues per +rule; binding constraint quantified: gate quality is worth ~9.6 overall +points (50.0 deployed vs 59.6 oracle). Also noted: the LUT re-run of the +canonical merge shows small systematic drift vs the Jul-13 eval (k4 hard +46.4 identical, k1/k2 hard 3 items lower) — the LUT (per-item, single +harness run) is now the canonical reference. Next candidates, in cost +order: (i) deploy E0's probe AS the gate against the LUT (free, +offline); (ii) stronger classifier (multi-position features, more data, +calibrated threshold); (iii) oracle-gap error analysis on the hard items +no fixed k solves but some k does. + +21. **E2 stage A: dense short-CoT supervision through the carry + whiteboard, GSM8K (pre-registered 2026-07-16 ~02:55, before running; + PLAN_SELFPACED E2 / the hybrid from the internalization discussion).** + Prep: harvest TERSE verified CoTs ("at most 3 short steps", answer- + verified, STaR filter) for GSM train. Arms: (A) carry regime + (k=2 prefill, pauses easy p=2 / hard p=6) trained with CE on + scratchpad+answer (~30-60 dense tokens — the ingredient every latent + GSM arm lacked); (B) CONTROL: identical supervision, feedforward + adapter, no recurrence. Eval: GSM test 256, grid 0:0 (base), 2:2, + 2:6; e400 checkpoints. Predictions: (a) arm A beats every previous + GSM arm's overall (>12.1%) — dense supervision is the binding fix; + (b) the A-vs-B delta isolates the whiteboard: if A > B by >=3 points + overall, recurrence adds value beyond visible-scratchpad training; + if A ~= B, the scratchpad text alone carries it (deflation, GSM + edition); (c) easy-bucket damage smaller than answer-only carry's + (83->45%) because training and deployment output formats now match. + Honest note: arm outputs are VISIBLE tokens (~40) — this is the + budget-CoT-with-loop hybrid, a scope change from latent planning, + run at Nils's explicit direction ("do gsm8k and such"). diff --git a/results-loop/eval_code_merge_lut.json b/results-loop/eval_code_merge_lut.json new file mode 100644 index 0000000..a5d8504 --- /dev/null +++ b/results-loop/eval_code_merge_lut.json @@ -0,0 +1,4046 @@ +{ + "tag": "merge_lut", + "ks": { + "0": { + "acc": 0.488, + "by_label": { + "easy": 0.9836065573770492, + "hard": 0.03571428571428571, + "drop": 0.01 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": false + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": true + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": true + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": false + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": false + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": false + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": false + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": false + }, + { + "task_id": 131, + "ok": true + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": true + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": false + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": false + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": false + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": true + }, + { + "task_id": 195, + "ok": false + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": false + }, + { + "task_id": 213, + "ok": true + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": true + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": false + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": true + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": true + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": false + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + } + ] + }, + "1": { + "acc": 0.472, + "by_label": { + "easy": 0.8688524590163934, + "hard": 0.32142857142857145, + "drop": 0.03 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": true + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": false + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": false + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": false + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": true + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": false + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": false + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": true + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": false + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": false + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": false + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": true + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": false + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": true + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": false + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": false + }, + { + "task_id": 131, + "ok": false + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": true + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": true + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": false + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": false + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": true + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": false + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": false + }, + { + "task_id": 195, + "ok": true + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": true + }, + { + "task_id": 213, + "ok": false + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": true + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": false + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": false + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": true + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": true + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + } + ] + }, + "2": { + "acc": 0.484, + "by_label": { + "easy": 0.8770491803278688, + "hard": 0.39285714285714285, + "drop": 0.03 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": false + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": true + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": false + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": true + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": false + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": true + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": false + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": true + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": false + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": true + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": true + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": false + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": true + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": false + }, + { + "task_id": 131, + "ok": false + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": false + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": false + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": false + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": false + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": false + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": true + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": false + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": false + }, + { + "task_id": 195, + "ok": true + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": true + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": true + }, + { + "task_id": 213, + "ok": false + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": false + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": true + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": false + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": false + }, + { + "task_id": 247, + "ok": true + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": true + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + } + ] + }, + "4": { + "acc": 0.52, + "by_label": { + "easy": 0.8934426229508197, + "hard": 0.4642857142857143, + "drop": 0.08 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": true + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": true + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": true + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": false + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": false + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": true + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": false + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": false + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": true + }, + { + "task_id": 73, + "ok": false + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": false + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": true + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": false + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": true + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": true + }, + { + "task_id": 131, + "ok": false + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": false + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": true + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": false + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": true + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": true + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": true + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": true + }, + { + "task_id": 194, + "ok": true + }, + { + "task_id": 195, + "ok": true + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": true + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": false + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": true + }, + { + "task_id": 213, + "ok": false + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": false + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": true + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": false + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": false + }, + { + "task_id": 247, + "ok": true + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": true + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": true + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": false + }, + { + "task_id": 259, + "ok": true + }, + { + "task_id": 260, + "ok": false + } + ] + } + }, + "n": 250 +} \ No newline at end of file diff --git a/results-loop/gate_probs.json b/results-loop/gate_probs.json new file mode 100644 index 0000000..a495ab5 --- /dev/null +++ b/results-loop/gate_probs.json @@ -0,0 +1,2752 @@ +[ + { + "task_id": 11, + "label": "easy", + "p": [ + 0.89357, + 0.89055, + 0.93474, + 0.9777, + 0.99153 + ] + }, + { + "task_id": 12, + "label": "hard", + "p": [ + 0.50398, + 0.30694, + 0.30154, + 0.48391, + 0.66596 + ] + }, + { + "task_id": 13, + "label": "drop", + "p": [ + 0.79083, + 0.83183, + 0.76197, + 0.91476, + 0.96338 + ] + }, + { + "task_id": 14, + "label": "easy", + "p": [ + 0.83086, + 0.92627, + 0.94583, + 0.95304, + 0.99234 + ] + }, + { + "task_id": 15, + "label": "drop", + "p": [ + 0.76689, + 0.67136, + 0.84591, + 0.94434, + 0.98057 + ] + }, + { + "task_id": 16, + "label": "hard", + "p": [ + 0.44951, + 0.55237, + 0.62362, + 0.85022, + 0.94957 + ] + }, + { + "task_id": 17, + "label": "easy", + "p": [ + 0.9937, + 0.99512, + 0.99711, + 0.99426, + 0.99811 + ] + }, + { + "task_id": 18, + "label": "easy", + "p": [ + 0.79653, + 0.84619, + 0.84163, + 0.96109, + 0.98216 + ] + }, + { + "task_id": 19, + "label": "easy", + "p": [ + 0.93109, + 0.9453, + 0.90862, + 0.96691, + 0.98738 + ] + }, + { + "task_id": 20, + "label": "drop", + "p": [ + 0.08349, + 0.27404, + 0.52305, + 0.68454, + 0.90872 + ] + }, + { + "task_id": 21, + "label": "drop", + "p": [ + 0.75473, + 0.84379, + 0.80387, + 0.91777, + 0.96119 + ] + }, + { + "task_id": 22, + "label": "easy", + "p": [ + 0.73855, + 0.72985, + 0.6788, + 0.88589, + 0.95621 + ] + }, + { + "task_id": 23, + "label": "easy", + "p": [ + 0.71153, + 0.7813, + 0.7143, + 0.93028, + 0.96614 + ] + }, + { + "task_id": 24, + "label": "drop", + "p": [ + 0.89665, + 0.96493, + 0.9516, + 0.96757, + 0.99089 + ] + }, + { + "task_id": 25, + "label": "easy", + "p": [ + 0.66187, + 0.62549, + 0.65389, + 0.86377, + 0.94826 + ] + }, + { + "task_id": 26, + "label": "drop", + "p": [ + 0.58546, + 0.67977, + 0.69435, + 0.8752, + 0.95235 + ] + }, + { + "task_id": 27, + "label": "easy", + "p": [ + 0.9664, + 0.96231, + 0.97402, + 0.98959, + 0.99638 + ] + }, + { + "task_id": 28, + "label": "easy", + "p": [ + 0.85238, + 0.8281, + 0.80848, + 0.90745, + 0.96387 + ] + }, + { + "task_id": 29, + "label": "easy", + "p": [ + 0.69342, + 0.79237, + 0.68813, + 0.90782, + 0.95878 + ] + }, + { + "task_id": 30, + "label": "easy", + "p": [ + 0.50807, + 0.57575, + 0.50629, + 0.83936, + 0.93659 + ] + }, + { + "task_id": 31, + "label": "drop", + "p": [ + 0.00581, + 0.01227, + 0.01094, + 0.0313, + 0.04651 + ] + }, + { + "task_id": 32, + "label": "easy", + "p": [ + 0.85777, + 0.78208, + 0.74828, + 0.87444, + 0.96034 + ] + }, + { + "task_id": 33, + "label": "drop", + "p": [ + 0.97348, + 0.97489, + 0.9669, + 0.98606, + 0.99539 + ] + }, + { + "task_id": 34, + "label": "drop", + "p": [ + 0.62812, + 0.70468, + 0.68783, + 0.90605, + 0.95461 + ] + }, + { + "task_id": 35, + "label": "drop", + "p": [ + 0.55073, + 0.46308, + 0.60041, + 0.70388, + 0.9282 + ] + }, + { + "task_id": 36, + "label": "drop", + "p": [ + 0.02165, + 0.04367, + 0.0976, + 0.177, + 0.46422 + ] + }, + { + "task_id": 37, + "label": "easy", + "p": [ + 0.76035, + 0.67788, + 0.6247, + 0.84582, + 0.92356 + ] + }, + { + "task_id": 38, + "label": "drop", + "p": [ + 0.33293, + 0.36407, + 0.40157, + 0.75703, + 0.90675 + ] + }, + { + "task_id": 39, + "label": "drop", + "p": [ + 0.15, + 0.30753, + 0.28027, + 0.67596, + 0.79183 + ] + }, + { + "task_id": 40, + "label": "easy", + "p": [ + 0.83272, + 0.80581, + 0.72565, + 0.88869, + 0.92634 + ] + }, + { + "task_id": 41, + "label": "easy", + "p": [ + 0.88886, + 0.93999, + 0.90242, + 0.95636, + 0.9743 + ] + }, + { + "task_id": 42, + "label": "drop", + "p": [ + 0.73807, + 0.87969, + 0.86408, + 0.95264, + 0.98347 + ] + }, + { + "task_id": 43, + "label": "drop", + "p": [ + 0.33375, + 0.48372, + 0.67104, + 0.846, + 0.93915 + ] + }, + { + "task_id": 44, + "label": "easy", + "p": [ + 0.67128, + 0.74514, + 0.85656, + 0.86085, + 0.9549 + ] + }, + { + "task_id": 45, + "label": "easy", + "p": [ + 0.85458, + 0.93449, + 0.88, + 0.92163, + 0.9783 + ] + }, + { + "task_id": 46, + "label": "easy", + "p": [ + 0.92276, + 0.93675, + 0.92815, + 0.9712, + 0.98908 + ] + }, + { + "task_id": 47, + "label": "drop", + "p": [ + 0.10422, + 0.14949, + 0.20522, + 0.38356, + 0.65836 + ] + }, + { + "task_id": 48, + "label": "drop", + "p": [ + 0.84921, + 0.84138, + 0.84399, + 0.95153, + 0.9849 + ] + }, + { + "task_id": 49, + "label": "easy", + "p": [ + 0.87909, + 0.87913, + 0.84868, + 0.94955, + 0.9795 + ] + }, + { + "task_id": 50, + "label": "easy", + "p": [ + 0.46225, + 0.46633, + 0.71266, + 0.84438, + 0.92548 + ] + }, + { + "task_id": 51, + "label": "easy", + "p": [ + 0.72267, + 0.72682, + 0.81249, + 0.82097, + 0.96662 + ] + }, + { + "task_id": 52, + "label": "easy", + "p": [ + 0.92985, + 0.96088, + 0.96472, + 0.96265, + 0.99004 + ] + }, + { + "task_id": 53, + "label": "easy", + "p": [ + 0.96517, + 0.97495, + 0.97593, + 0.98896, + 0.99623 + ] + }, + { + "task_id": 54, + "label": "hard", + "p": [ + 0.2274, + 0.09455, + 0.11943, + 0.28229, + 0.46073 + ] + }, + { + "task_id": 55, + "label": "easy", + "p": [ + 0.64737, + 0.72023, + 0.72777, + 0.81228, + 0.96691 + ] + }, + { + "task_id": 56, + "label": "easy", + "p": [ + 0.71643, + 0.78519, + 0.76763, + 0.9094, + 0.97177 + ] + }, + { + "task_id": 57, + "label": "easy", + "p": [ + 0.88177, + 0.84966, + 0.84267, + 0.94986, + 0.97897 + ] + }, + { + "task_id": 58, + "label": "easy", + "p": [ + 0.98071, + 0.98204, + 0.98632, + 0.98348, + 0.99592 + ] + }, + { + "task_id": 59, + "label": "hard", + "p": [ + 0.25404, + 0.27921, + 0.54846, + 0.72444, + 0.92613 + ] + }, + { + "task_id": 60, + "label": "drop", + "p": [ + 0.14619, + 0.24682, + 0.13947, + 0.309, + 0.51673 + ] + }, + { + "task_id": 61, + "label": "easy", + "p": [ + 0.11802, + 0.19595, + 0.2016, + 0.59332, + 0.75539 + ] + }, + { + "task_id": 62, + "label": "easy", + "p": [ + 0.9885, + 0.98971, + 0.984, + 0.99164, + 0.99629 + ] + }, + { + "task_id": 63, + "label": "easy", + "p": [ + 0.68078, + 0.64233, + 0.69578, + 0.89857, + 0.9497 + ] + }, + { + "task_id": 64, + "label": "easy", + "p": [ + 0.81591, + 0.84676, + 0.80416, + 0.90977, + 0.95095 + ] + }, + { + "task_id": 65, + "label": "easy", + "p": [ + 0.57192, + 0.6689, + 0.57318, + 0.80944, + 0.94089 + ] + }, + { + "task_id": 66, + "label": "easy", + "p": [ + 0.98745, + 0.99131, + 0.99017, + 0.99263, + 0.99801 + ] + }, + { + "task_id": 67, + "label": "drop", + "p": [ + 0.01655, + 0.05309, + 0.09597, + 0.18956, + 0.41828 + ] + }, + { + "task_id": 68, + "label": "easy", + "p": [ + 0.48384, + 0.28398, + 0.28519, + 0.51157, + 0.77851 + ] + }, + { + "task_id": 69, + "label": "easy", + "p": [ + 0.81234, + 0.80784, + 0.77323, + 0.88116, + 0.95919 + ] + }, + { + "task_id": 70, + "label": "easy", + "p": [ + 0.87639, + 0.8824, + 0.86506, + 0.96868, + 0.98536 + ] + }, + { + "task_id": 71, + "label": "hard", + "p": [ + 0.04806, + 0.04298, + 0.14249, + 0.19625, + 0.38761 + ] + }, + { + "task_id": 72, + "label": "hard", + "p": [ + 0.32363, + 0.23001, + 0.25665, + 0.4381, + 0.65388 + ] + }, + { + "task_id": 73, + "label": "hard", + "p": [ + 0.73292, + 0.75375, + 0.74466, + 0.95447, + 0.97219 + ] + }, + { + "task_id": 74, + "label": "drop", + "p": [ + 0.11047, + 0.18121, + 0.20217, + 0.51955, + 0.77091 + ] + }, + { + "task_id": 75, + "label": "easy", + "p": [ + 0.78349, + 0.74908, + 0.66469, + 0.93091, + 0.96677 + ] + }, + { + "task_id": 76, + "label": "easy", + "p": [ + 0.28597, + 0.47215, + 0.5713, + 0.73341, + 0.93328 + ] + }, + { + "task_id": 77, + "label": "drop", + "p": [ + 0.41215, + 0.54066, + 0.66755, + 0.79455, + 0.92596 + ] + }, + { + "task_id": 78, + "label": "easy", + "p": [ + 0.39123, + 0.49212, + 0.39897, + 0.715, + 0.87077 + ] + }, + { + "task_id": 79, + "label": "easy", + "p": [ + 0.9765, + 0.98781, + 0.99213, + 0.9895, + 0.99665 + ] + }, + { + "task_id": 80, + "label": "easy", + "p": [ + 0.78536, + 0.62279, + 0.69841, + 0.79474, + 0.95346 + ] + }, + { + "task_id": 81, + "label": "drop", + "p": [ + 0.9395, + 0.98022, + 0.97141, + 0.99033, + 0.99527 + ] + }, + { + "task_id": 82, + "label": "easy", + "p": [ + 0.97937, + 0.98545, + 0.98164, + 0.99252, + 0.99769 + ] + }, + { + "task_id": 83, + "label": "drop", + "p": [ + 0.1951, + 0.48442, + 0.64875, + 0.87044, + 0.94897 + ] + }, + { + "task_id": 84, + "label": "drop", + "p": [ + 0.14927, + 0.17933, + 0.2864, + 0.44782, + 0.78894 + ] + }, + { + "task_id": 85, + "label": "easy", + "p": [ + 0.98233, + 0.98519, + 0.9834, + 0.99125, + 0.99692 + ] + }, + { + "task_id": 86, + "label": "easy", + "p": [ + 0.47477, + 0.35897, + 0.51764, + 0.65309, + 0.89107 + ] + }, + { + "task_id": 87, + "label": "drop", + "p": [ + 0.95619, + 0.9403, + 0.92605, + 0.96954, + 0.98269 + ] + }, + { + "task_id": 88, + "label": "easy", + "p": [ + 0.97505, + 0.97184, + 0.95207, + 0.98726, + 0.99441 + ] + }, + { + "task_id": 89, + "label": "easy", + "p": [ + 0.80577, + 0.71943, + 0.77858, + 0.86369, + 0.95698 + ] + }, + { + "task_id": 90, + "label": "easy", + "p": [ + 0.93165, + 0.95475, + 0.95756, + 0.98427, + 0.99385 + ] + }, + { + "task_id": 91, + "label": "easy", + "p": [ + 0.91589, + 0.94138, + 0.92554, + 0.97551, + 0.9897 + ] + }, + { + "task_id": 92, + "label": "hard", + "p": [ + 0.25405, + 0.19562, + 0.3128, + 0.54556, + 0.79543 + ] + }, + { + "task_id": 93, + "label": "easy", + "p": [ + 0.96386, + 0.97946, + 0.97682, + 0.9748, + 0.98987 + ] + }, + { + "task_id": 94, + "label": "easy", + "p": [ + 0.82638, + 0.80919, + 0.73887, + 0.9282, + 0.9754 + ] + }, + { + "task_id": 95, + "label": "drop", + "p": [ + 0.46674, + 0.47501, + 0.50147, + 0.82917, + 0.94048 + ] + }, + { + "task_id": 96, + "label": "easy", + "p": [ + 0.91732, + 0.91352, + 0.89195, + 0.95806, + 0.98139 + ] + }, + { + "task_id": 97, + "label": "easy", + "p": [ + 0.85491, + 0.85131, + 0.82494, + 0.9345, + 0.95773 + ] + }, + { + "task_id": 98, + "label": "easy", + "p": [ + 0.94755, + 0.9692, + 0.95996, + 0.98385, + 0.99405 + ] + }, + { + "task_id": 99, + "label": "easy", + "p": [ + 0.97298, + 0.97692, + 0.96106, + 0.98073, + 0.99426 + ] + }, + { + "task_id": 100, + "label": "drop", + "p": [ + 0.22422, + 0.16655, + 0.18838, + 0.3867, + 0.64838 + ] + }, + { + "task_id": 101, + "label": "drop", + "p": [ + 0.51609, + 0.56818, + 0.54542, + 0.86378, + 0.94497 + ] + }, + { + "task_id": 102, + "label": "easy", + "p": [ + 0.917, + 0.87774, + 0.87039, + 0.90651, + 0.96996 + ] + }, + { + "task_id": 103, + "label": "hard", + "p": [ + 0.07738, + 0.0892, + 0.29045, + 0.46846, + 0.78097 + ] + }, + { + "task_id": 104, + "label": "easy", + "p": [ + 0.79425, + 0.73746, + 0.86208, + 0.93078, + 0.95346 + ] + }, + { + "task_id": 105, + "label": "easy", + "p": [ + 0.99419, + 0.99647, + 0.99715, + 0.99674, + 0.99898 + ] + }, + { + "task_id": 106, + "label": "drop", + "p": [ + 0.94962, + 0.93781, + 0.9476, + 0.98224, + 0.99193 + ] + }, + { + "task_id": 107, + "label": "drop", + "p": [ + 0.59585, + 0.55188, + 0.73479, + 0.86862, + 0.95316 + ] + }, + { + "task_id": 108, + "label": "drop", + "p": [ + 0.0696, + 0.05663, + 0.09562, + 0.14792, + 0.19583 + ] + }, + { + "task_id": 109, + "label": "hard", + "p": [ + 0.119, + 0.22441, + 0.36733, + 0.76221, + 0.90479 + ] + }, + { + "task_id": 110, + "label": "drop", + "p": [ + 0.20118, + 0.33392, + 0.44287, + 0.55401, + 0.7197 + ] + }, + { + "task_id": 111, + "label": "drop", + "p": [ + 0.68501, + 0.67557, + 0.67574, + 0.88966, + 0.92607 + ] + }, + { + "task_id": 112, + "label": "drop", + "p": [ + 0.7953, + 0.9301, + 0.96948, + 0.9645, + 0.99465 + ] + }, + { + "task_id": 113, + "label": "easy", + "p": [ + 0.86993, + 0.85243, + 0.81632, + 0.89736, + 0.96394 + ] + }, + { + "task_id": 114, + "label": "drop", + "p": [ + 0.63413, + 0.61198, + 0.74885, + 0.8811, + 0.94821 + ] + }, + { + "task_id": 115, + "label": "easy", + "p": [ + 0.90051, + 0.92177, + 0.95171, + 0.96419, + 0.99066 + ] + }, + { + "task_id": 116, + "label": "easy", + "p": [ + 0.98341, + 0.98105, + 0.97568, + 0.98362, + 0.99452 + ] + }, + { + "task_id": 117, + "label": "easy", + "p": [ + 0.6718, + 0.68576, + 0.66937, + 0.88347, + 0.95018 + ] + }, + { + "task_id": 118, + "label": "hard", + "p": [ + 0.90568, + 0.92727, + 0.96231, + 0.97739, + 0.98882 + ] + }, + { + "task_id": 119, + "label": "drop", + "p": [ + 0.27799, + 0.37071, + 0.39517, + 0.73715, + 0.8783 + ] + }, + { + "task_id": 120, + "label": "drop", + "p": [ + 0.58452, + 0.49425, + 0.50749, + 0.80844, + 0.929 + ] + }, + { + "task_id": 121, + "label": "drop", + "p": [ + 0.17097, + 0.26309, + 0.17445, + 0.32482, + 0.55714 + ] + }, + { + "task_id": 122, + "label": "drop", + "p": [ + 0.10363, + 0.09689, + 0.18023, + 0.28794, + 0.67192 + ] + }, + { + "task_id": 123, + "label": "drop", + "p": [ + 0.23776, + 0.31546, + 0.24108, + 0.41142, + 0.65403 + ] + }, + { + "task_id": 124, + "label": "drop", + "p": [ + 0.90079, + 0.92766, + 0.87525, + 0.95659, + 0.98826 + ] + }, + { + "task_id": 125, + "label": "drop", + "p": [ + 0.36597, + 0.39777, + 0.16709, + 0.55427, + 0.76332 + ] + }, + { + "task_id": 126, + "label": "drop", + "p": [ + 0.47293, + 0.67885, + 0.73117, + 0.86132, + 0.93284 + ] + }, + { + "task_id": 127, + "label": "easy", + "p": [ + 0.47397, + 0.4237, + 0.39502, + 0.58215, + 0.8086 + ] + }, + { + "task_id": 128, + "label": "easy", + "p": [ + 0.9321, + 0.91941, + 0.94504, + 0.96873, + 0.98926 + ] + }, + { + "task_id": 129, + "label": "hard", + "p": [ + 0.05544, + 0.0764, + 0.10903, + 0.22014, + 0.35951 + ] + }, + { + "task_id": 130, + "label": "drop", + "p": [ + 0.77887, + 0.74506, + 0.76802, + 0.90183, + 0.95243 + ] + }, + { + "task_id": 131, + "label": "easy", + "p": [ + 0.86709, + 0.8504, + 0.8476, + 0.93274, + 0.98341 + ] + }, + { + "task_id": 132, + "label": "easy", + "p": [ + 0.97743, + 0.98532, + 0.97986, + 0.99262, + 0.99693 + ] + }, + { + "task_id": 133, + "label": "easy", + "p": [ + 0.8991, + 0.92459, + 0.89804, + 0.94099, + 0.96291 + ] + }, + { + "task_id": 134, + "label": "drop", + "p": [ + 0.14039, + 0.3363, + 0.33071, + 0.63899, + 0.85239 + ] + }, + { + "task_id": 135, + "label": "easy", + "p": [ + 0.66224, + 0.5986, + 0.70907, + 0.78321, + 0.94245 + ] + }, + { + "task_id": 136, + "label": "drop", + "p": [ + 0.39408, + 0.5581, + 0.77356, + 0.79034, + 0.95849 + ] + }, + { + "task_id": 137, + "label": "drop", + "p": [ + 0.31717, + 0.30142, + 0.3008, + 0.60465, + 0.78387 + ] + }, + { + "task_id": 138, + "label": "drop", + "p": [ + 0.26625, + 0.28838, + 0.20698, + 0.49102, + 0.68148 + ] + }, + { + "task_id": 139, + "label": "drop", + "p": [ + 0.96798, + 0.97592, + 0.98152, + 0.98589, + 0.99656 + ] + }, + { + "task_id": 140, + "label": "drop", + "p": [ + 0.69138, + 0.6692, + 0.7287, + 0.86477, + 0.93824 + ] + }, + { + "task_id": 141, + "label": "hard", + "p": [ + 0.15042, + 0.04784, + 0.08953, + 0.09043, + 0.17728 + ] + }, + { + "task_id": 142, + "label": "drop", + "p": [ + 0.07279, + 0.11019, + 0.08753, + 0.24872, + 0.5145 + ] + }, + { + "task_id": 143, + "label": "drop", + "p": [ + 0.64137, + 0.62778, + 0.62918, + 0.87075, + 0.9459 + ] + }, + { + "task_id": 144, + "label": "easy", + "p": [ + 0.53405, + 0.69171, + 0.62357, + 0.88579, + 0.94673 + ] + }, + { + "task_id": 145, + "label": "easy", + "p": [ + 0.9411, + 0.95735, + 0.93679, + 0.98149, + 0.99309 + ] + }, + { + "task_id": 146, + "label": "drop", + "p": [ + 0.91423, + 0.95228, + 0.97015, + 0.97533, + 0.99193 + ] + }, + { + "task_id": 147, + "label": "drop", + "p": [ + 0.17211, + 0.33202, + 0.11193, + 0.45544, + 0.6481 + ] + }, + { + "task_id": 148, + "label": "drop", + "p": [ + 0.28321, + 0.29185, + 0.36127, + 0.62591, + 0.84094 + ] + }, + { + "task_id": 149, + "label": "drop", + "p": [ + 0.04267, + 0.08807, + 0.04533, + 0.14966, + 0.30373 + ] + }, + { + "task_id": 150, + "label": "drop", + "p": [ + 0.03545, + 0.14585, + 0.20081, + 0.36988, + 0.72517 + ] + }, + { + "task_id": 151, + "label": "easy", + "p": [ + 0.83287, + 0.88096, + 0.80491, + 0.89386, + 0.96534 + ] + }, + { + "task_id": 152, + "label": "hard", + "p": [ + 0.56426, + 0.27253, + 0.27571, + 0.53634, + 0.66257 + ] + }, + { + "task_id": 153, + "label": "easy", + "p": [ + 0.62975, + 0.82231, + 0.67015, + 0.87465, + 0.97254 + ] + }, + { + "task_id": 154, + "label": "easy", + "p": [ + 0.91225, + 0.93355, + 0.88878, + 0.96774, + 0.98652 + ] + }, + { + "task_id": 155, + "label": "drop", + "p": [ + 0.72667, + 0.79122, + 0.73712, + 0.93298, + 0.97292 + ] + }, + { + "task_id": 156, + "label": "easy", + "p": [ + 0.97981, + 0.97924, + 0.97651, + 0.99278, + 0.99704 + ] + }, + { + "task_id": 157, + "label": "easy", + "p": [ + 0.61519, + 0.47275, + 0.56045, + 0.79935, + 0.892 + ] + }, + { + "task_id": 158, + "label": "drop", + "p": [ + 0.04778, + 0.06587, + 0.11013, + 0.30719, + 0.55525 + ] + }, + { + "task_id": 159, + "label": "drop", + "p": [ + 0.71601, + 0.76572, + 0.89768, + 0.93356, + 0.9823 + ] + }, + { + "task_id": 160, + "label": "drop", + "p": [ + 0.28528, + 0.31888, + 0.26141, + 0.64715, + 0.83732 + ] + }, + { + "task_id": 161, + "label": "easy", + "p": [ + 0.88818, + 0.88305, + 0.85268, + 0.96653, + 0.97963 + ] + }, + { + "task_id": 162, + "label": "easy", + "p": [ + 0.61639, + 0.66463, + 0.62924, + 0.87766, + 0.95297 + ] + }, + { + "task_id": 163, + "label": "easy", + "p": [ + 0.81961, + 0.84259, + 0.8152, + 0.92246, + 0.97728 + ] + }, + { + "task_id": 164, + "label": "drop", + "p": [ + 0.15, + 0.29692, + 0.40041, + 0.62579, + 0.85607 + ] + }, + { + "task_id": 165, + "label": "drop", + "p": [ + 0.58623, + 0.60073, + 0.69265, + 0.88662, + 0.95531 + ] + }, + { + "task_id": 166, + "label": "hard", + "p": [ + 0.25528, + 0.387, + 0.31115, + 0.68663, + 0.83833 + ] + }, + { + "task_id": 167, + "label": "easy", + "p": [ + 0.91445, + 0.87081, + 0.83912, + 0.93394, + 0.96386 + ] + }, + { + "task_id": 168, + "label": "easy", + "p": [ + 0.96198, + 0.96478, + 0.95719, + 0.98187, + 0.99353 + ] + }, + { + "task_id": 169, + "label": "drop", + "p": [ + 0.67866, + 0.69527, + 0.73525, + 0.81971, + 0.94709 + ] + }, + { + "task_id": 170, + "label": "hard", + "p": [ + 0.95203, + 0.93717, + 0.90739, + 0.97571, + 0.98717 + ] + }, + { + "task_id": 171, + "label": "easy", + "p": [ + 0.92192, + 0.95594, + 0.96209, + 0.95983, + 0.98959 + ] + }, + { + "task_id": 172, + "label": "easy", + "p": [ + 0.84575, + 0.90533, + 0.91592, + 0.96722, + 0.98685 + ] + }, + { + "task_id": 173, + "label": "easy", + "p": [ + 0.97814, + 0.97659, + 0.96504, + 0.98998, + 0.99487 + ] + }, + { + "task_id": 174, + "label": "easy", + "p": [ + 0.96537, + 0.96209, + 0.94894, + 0.98345, + 0.99146 + ] + }, + { + "task_id": 175, + "label": "easy", + "p": [ + 0.68994, + 0.71995, + 0.68183, + 0.84518, + 0.95452 + ] + }, + { + "task_id": 176, + "label": "easy", + "p": [ + 0.98563, + 0.98475, + 0.97968, + 0.98554, + 0.99604 + ] + }, + { + "task_id": 177, + "label": "drop", + "p": [ + 0.05297, + 0.08146, + 0.18211, + 0.32539, + 0.56394 + ] + }, + { + "task_id": 178, + "label": "easy", + "p": [ + 0.36335, + 0.37762, + 0.52415, + 0.74358, + 0.89633 + ] + }, + { + "task_id": 179, + "label": "easy", + "p": [ + 0.06798, + 0.15871, + 0.32543, + 0.5342, + 0.82607 + ] + }, + { + "task_id": 180, + "label": "drop", + "p": [ + 0.62057, + 0.8147, + 0.8733, + 0.94409, + 0.97591 + ] + }, + { + "task_id": 181, + "label": "drop", + "p": [ + 0.66807, + 0.79339, + 0.69337, + 0.8994, + 0.96273 + ] + }, + { + "task_id": 182, + "label": "drop", + "p": [ + 0.3612, + 0.26238, + 0.5549, + 0.88377, + 0.93233 + ] + }, + { + "task_id": 183, + "label": "drop", + "p": [ + 0.28266, + 0.29534, + 0.16807, + 0.51871, + 0.72209 + ] + }, + { + "task_id": 184, + "label": "drop", + "p": [ + 0.87555, + 0.89862, + 0.94034, + 0.9541, + 0.98513 + ] + }, + { + "task_id": 185, + "label": "drop", + "p": [ + 0.29228, + 0.49984, + 0.48585, + 0.73878, + 0.91567 + ] + }, + { + "task_id": 186, + "label": "easy", + "p": [ + 0.58891, + 0.58056, + 0.68445, + 0.88125, + 0.95079 + ] + }, + { + "task_id": 187, + "label": "hard", + "p": [ + 0.28565, + 0.49773, + 0.2114, + 0.56563, + 0.77521 + ] + }, + { + "task_id": 188, + "label": "drop", + "p": [ + 0.18024, + 0.28034, + 0.30994, + 0.57573, + 0.77738 + ] + }, + { + "task_id": 189, + "label": "hard", + "p": [ + 0.43512, + 0.45572, + 0.50314, + 0.75548, + 0.89447 + ] + }, + { + "task_id": 190, + "label": "drop", + "p": [ + 0.68482, + 0.75532, + 0.68872, + 0.8655, + 0.94554 + ] + }, + { + "task_id": 191, + "label": "easy", + "p": [ + 0.71228, + 0.72672, + 0.84377, + 0.89845, + 0.96688 + ] + }, + { + "task_id": 192, + "label": "easy", + "p": [ + 0.94378, + 0.95396, + 0.96194, + 0.97179, + 0.99306 + ] + }, + { + "task_id": 193, + "label": "drop", + "p": [ + 0.95737, + 0.963, + 0.93888, + 0.99207, + 0.99502 + ] + }, + { + "task_id": 194, + "label": "easy", + "p": [ + 0.95924, + 0.97098, + 0.96623, + 0.98802, + 0.99547 + ] + }, + { + "task_id": 195, + "label": "hard", + "p": [ + 0.72865, + 0.80805, + 0.84713, + 0.93955, + 0.97687 + ] + }, + { + "task_id": 196, + "label": "easy", + "p": [ + 0.92579, + 0.91489, + 0.88756, + 0.969, + 0.98627 + ] + }, + { + "task_id": 197, + "label": "easy", + "p": [ + 0.56574, + 0.75574, + 0.68648, + 0.88917, + 0.9496 + ] + }, + { + "task_id": 198, + "label": "drop", + "p": [ + 0.05324, + 0.0904, + 0.0964, + 0.15749, + 0.39498 + ] + }, + { + "task_id": 199, + "label": "easy", + "p": [ + 0.92675, + 0.92159, + 0.86901, + 0.95232, + 0.97729 + ] + }, + { + "task_id": 200, + "label": "easy", + "p": [ + 0.86798, + 0.7788, + 0.79958, + 0.94818, + 0.97705 + ] + }, + { + "task_id": 201, + "label": "easy", + "p": [ + 0.82087, + 0.86468, + 0.88409, + 0.92087, + 0.97416 + ] + }, + { + "task_id": 202, + "label": "easy", + "p": [ + 0.95307, + 0.96821, + 0.9693, + 0.98233, + 0.99467 + ] + }, + { + "task_id": 203, + "label": "easy", + "p": [ + 0.42132, + 0.67164, + 0.72252, + 0.81074, + 0.93495 + ] + }, + { + "task_id": 204, + "label": "easy", + "p": [ + 0.99286, + 0.99543, + 0.99293, + 0.9969, + 0.99874 + ] + }, + { + "task_id": 205, + "label": "drop", + "p": [ + 0.35913, + 0.38246, + 0.40243, + 0.68678, + 0.81736 + ] + }, + { + "task_id": 206, + "label": "drop", + "p": [ + 0.89634, + 0.84776, + 0.82767, + 0.96446, + 0.98386 + ] + }, + { + "task_id": 207, + "label": "drop", + "p": [ + 0.00393, + 0.01615, + 0.01426, + 0.04742, + 0.10254 + ] + }, + { + "task_id": 208, + "label": "hard", + "p": [ + 0.31202, + 0.3439, + 0.65669, + 0.75873, + 0.90626 + ] + }, + { + "task_id": 209, + "label": "drop", + "p": [ + 0.21682, + 0.12314, + 0.15341, + 0.2044, + 0.29576 + ] + }, + { + "task_id": 210, + "label": "easy", + "p": [ + 0.61656, + 0.60869, + 0.80404, + 0.89388, + 0.93799 + ] + }, + { + "task_id": 211, + "label": "drop", + "p": [ + 0.59513, + 0.59442, + 0.62858, + 0.84734, + 0.95821 + ] + }, + { + "task_id": 212, + "label": "hard", + "p": [ + 0.85699, + 0.91676, + 0.861, + 0.94062, + 0.98022 + ] + }, + { + "task_id": 213, + "label": "drop", + "p": [ + 0.93777, + 0.91526, + 0.9085, + 0.97975, + 0.99035 + ] + }, + { + "task_id": 214, + "label": "easy", + "p": [ + 0.96082, + 0.97692, + 0.98235, + 0.99365, + 0.99699 + ] + }, + { + "task_id": 215, + "label": "hard", + "p": [ + 0.14916, + 0.16749, + 0.22751, + 0.44494, + 0.69792 + ] + }, + { + "task_id": 216, + "label": "drop", + "p": [ + 0.21699, + 0.18481, + 0.29528, + 0.49903, + 0.69645 + ] + }, + { + "task_id": 217, + "label": "easy", + "p": [ + 0.77553, + 0.81543, + 0.81827, + 0.94872, + 0.97779 + ] + }, + { + "task_id": 218, + "label": "drop", + "p": [ + 0.32571, + 0.39545, + 0.48954, + 0.69552, + 0.90008 + ] + }, + { + "task_id": 219, + "label": "drop", + "p": [ + 0.77409, + 0.73817, + 0.78044, + 0.94796, + 0.97244 + ] + }, + { + "task_id": 220, + "label": "drop", + "p": [ + 0.22025, + 0.22586, + 0.29641, + 0.56467, + 0.74762 + ] + }, + { + "task_id": 221, + "label": "easy", + "p": [ + 0.98675, + 0.99186, + 0.98327, + 0.99429, + 0.99737 + ] + }, + { + "task_id": 222, + "label": "easy", + "p": [ + 0.9193, + 0.89804, + 0.89969, + 0.92242, + 0.97512 + ] + }, + { + "task_id": 223, + "label": "hard", + "p": [ + 0.28705, + 0.28649, + 0.20383, + 0.38623, + 0.65183 + ] + }, + { + "task_id": 224, + "label": "easy", + "p": [ + 0.96567, + 0.96474, + 0.94717, + 0.97949, + 0.9922 + ] + }, + { + "task_id": 225, + "label": "hard", + "p": [ + 0.45459, + 0.55311, + 0.45331, + 0.74874, + 0.91006 + ] + }, + { + "task_id": 226, + "label": "easy", + "p": [ + 0.98014, + 0.97361, + 0.96906, + 0.99103, + 0.99608 + ] + }, + { + "task_id": 227, + "label": "easy", + "p": [ + 0.98421, + 0.98599, + 0.9888, + 0.98295, + 0.99445 + ] + }, + { + "task_id": 228, + "label": "hard", + "p": [ + 0.17152, + 0.36409, + 0.37197, + 0.67238, + 0.86486 + ] + }, + { + "task_id": 229, + "label": "drop", + "p": [ + 0.86396, + 0.75272, + 0.64182, + 0.91116, + 0.93053 + ] + }, + { + "task_id": 230, + "label": "easy", + "p": [ + 0.96772, + 0.97436, + 0.98073, + 0.98496, + 0.99499 + ] + }, + { + "task_id": 231, + "label": "drop", + "p": [ + 0.20273, + 0.39621, + 0.25285, + 0.6548, + 0.85831 + ] + }, + { + "task_id": 232, + "label": "easy", + "p": [ + 0.90432, + 0.89781, + 0.89127, + 0.94723, + 0.97067 + ] + }, + { + "task_id": 233, + "label": "drop", + "p": [ + 0.95418, + 0.96967, + 0.95252, + 0.97107, + 0.99265 + ] + }, + { + "task_id": 234, + "label": "easy", + "p": [ + 0.99496, + 0.99554, + 0.99757, + 0.99536, + 0.99854 + ] + }, + { + "task_id": 235, + "label": "drop", + "p": [ + 0.72484, + 0.74825, + 0.76061, + 0.91065, + 0.97662 + ] + }, + { + "task_id": 236, + "label": "drop", + "p": [ + 0.02384, + 0.03292, + 0.05297, + 0.13374, + 0.3091 + ] + }, + { + "task_id": 237, + "label": "drop", + "p": [ + 0.23048, + 0.41145, + 0.33877, + 0.64411, + 0.79658 + ] + }, + { + "task_id": 238, + "label": "easy", + "p": [ + 0.698, + 0.72427, + 0.50821, + 0.85209, + 0.94318 + ] + }, + { + "task_id": 239, + "label": "drop", + "p": [ + 0.046, + 0.09851, + 0.0578, + 0.19774, + 0.45162 + ] + }, + { + "task_id": 240, + "label": "easy", + "p": [ + 0.90875, + 0.93379, + 0.9566, + 0.98461, + 0.99168 + ] + }, + { + "task_id": 241, + "label": "drop", + "p": [ + 0.51053, + 0.69063, + 0.72965, + 0.96268, + 0.97812 + ] + }, + { + "task_id": 242, + "label": "easy", + "p": [ + 0.99177, + 0.99585, + 0.99661, + 0.99588, + 0.99859 + ] + }, + { + "task_id": 243, + "label": "drop", + "p": [ + 0.21756, + 0.29908, + 0.50762, + 0.75442, + 0.87674 + ] + }, + { + "task_id": 244, + "label": "easy", + "p": [ + 0.93607, + 0.89947, + 0.88266, + 0.95279, + 0.98299 + ] + }, + { + "task_id": 245, + "label": "drop", + "p": [ + 0.01681, + 0.02526, + 0.01987, + 0.06302, + 0.15793 + ] + }, + { + "task_id": 246, + "label": "easy", + "p": [ + 0.67901, + 0.57826, + 0.57164, + 0.73882, + 0.89346 + ] + }, + { + "task_id": 247, + "label": "hard", + "p": [ + 0.09516, + 0.18852, + 0.07139, + 0.33163, + 0.51497 + ] + }, + { + "task_id": 248, + "label": "drop", + "p": [ + 0.77539, + 0.79222, + 0.78893, + 0.87557, + 0.96437 + ] + }, + { + "task_id": 249, + "label": "drop", + "p": [ + 0.4796, + 0.49534, + 0.58499, + 0.79981, + 0.89051 + ] + }, + { + "task_id": 250, + "label": "easy", + "p": [ + 0.98285, + 0.9863, + 0.97883, + 0.99475, + 0.99742 + ] + }, + { + "task_id": 251, + "label": "easy", + "p": [ + 0.95647, + 0.96143, + 0.9512, + 0.97891, + 0.99172 + ] + }, + { + "task_id": 252, + "label": "easy", + "p": [ + 0.85514, + 0.90221, + 0.87323, + 0.93956, + 0.98677 + ] + }, + { + "task_id": 253, + "label": "easy", + "p": [ + 0.91462, + 0.9329, + 0.94117, + 0.95418, + 0.99017 + ] + }, + { + "task_id": 254, + "label": "easy", + "p": [ + 0.56536, + 0.78083, + 0.90357, + 0.93935, + 0.98103 + ] + }, + { + "task_id": 255, + "label": "drop", + "p": [ + 0.45615, + 0.54718, + 0.38428, + 0.7677, + 0.864 + ] + }, + { + "task_id": 256, + "label": "hard", + "p": [ + 0.70659, + 0.65657, + 0.57063, + 0.89185, + 0.94517 + ] + }, + { + "task_id": 257, + "label": "easy", + "p": [ + 0.97289, + 0.98164, + 0.98335, + 0.98048, + 0.99281 + ] + }, + { + "task_id": 258, + "label": "hard", + "p": [ + 0.87581, + 0.94185, + 0.87132, + 0.92484, + 0.96883 + ] + }, + { + "task_id": 259, + "label": "drop", + "p": [ + 0.40289, + 0.57161, + 0.6051, + 0.76364, + 0.86928 + ] + }, + { + "task_id": 260, + "label": "drop", + "p": [ + 0.01846, + 0.03097, + 0.15015, + 0.20159, + 0.53308 + ] + } +] \ No newline at end of file diff --git a/results-loop/gate_threshold_curve.json b/results-loop/gate_threshold_curve.json new file mode 100644 index 0000000..f326efb --- /dev/null +++ b/results-loop/gate_threshold_curve.json @@ -0,0 +1,93 @@ +{ + "curve": [ + { + "theta": 0.3, + "overall": 0.488, + "by_label": { + "easy": 0.9672131147540983, + "hard": 0.07142857142857142, + "drop": 0.02 + }, + "ek": 0.432 + }, + { + "theta": 0.5, + "overall": 0.5, + "by_label": { + "easy": 0.9590163934426229, + "hard": 0.17857142857142858, + "drop": 0.03 + }, + "ek": 0.74 + }, + { + "theta": 0.7, + "overall": 0.496, + "by_label": { + "easy": 0.9426229508196722, + "hard": 0.17857142857142858, + "drop": 0.04 + }, + "ek": 1.136 + }, + { + "theta": 0.8, + "overall": 0.508, + "by_label": { + "easy": 0.9344262295081968, + "hard": 0.2857142857142857, + "drop": 0.05 + }, + "ek": 1.388 + }, + { + "theta": 0.9, + "overall": 0.496, + "by_label": { + "easy": 0.8852459016393442, + "hard": 0.39285714285714285, + "drop": 0.05 + }, + "ek": 1.76 + }, + { + "theta": 0.95, + "overall": 0.492, + "by_label": { + "easy": 0.8770491803278688, + "hard": 0.42857142857142855, + "drop": 0.04 + }, + "ek": 2.084 + }, + { + "theta": 0.98, + "overall": 0.504, + "by_label": { + "easy": 0.8770491803278688, + "hard": 0.4642857142857143, + "drop": 0.06 + }, + "ek": 2.412 + }, + { + "theta": 0.99, + "overall": 0.508, + "by_label": { + "easy": 0.8770491803278688, + "hard": 0.5, + "drop": 0.06 + }, + "ek": 2.628 + } + ], + "oracle": { + "overall": 0.596, + "by_label": { + "easy": 1.0, + "hard": 0.6428571428571429, + "drop": 0.09 + }, + "ek": 0.236 + } +} \ No newline at end of file diff --git a/scripts/eval_carry_cot.py b/scripts/eval_carry_cot.py new file mode 100644 index 0000000..bd48d22 --- /dev/null +++ b/scripts/eval_carry_cot.py @@ -0,0 +1,94 @@ +"""E2 stage A eval (item 21): GSM8K accuracy for short-CoT-trained arms. + +Generates with generate_carry_c (prefill loop k + pause carry + per-token +carry) or feedforward mode for the control arm; scores last_number vs +gold, by STaR label. Grid over (k,p) cells. +""" + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +import torch + +from carry_common import generate_carry_c +from loop_common import (BandLooper, MergeAdapter, chat_prompt, + DIRECT_SUFFIX, last_number, num_eq) + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from jlens.core import load_model # noqa: E402 + +OUT = Path(os.environ.get("LOOP_OUT", + Path(__file__).resolve().parent.parent / "results-loop")) + + +@torch.no_grad() +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--adapter", required=True) + ap.add_argument("--tag", required=True) + ap.add_argument("--grid", default="0:0,2:2,2:6", + help="comma list of k:p cells") + ap.add_argument("--n", type=int, default=256) + ap.add_argument("--batch", type=int, default=8) + ap.add_argument("--feedforward", action="store_true") + ap.add_argument("--max-new", type=int, default=160) + args = ap.parse_args() + + model, tok = load_model(dtype=torch.bfloat16) + tok.padding_side = "left" + looper = BandLooper(model) + adapter = MergeAdapter( + d=model.config.get_text_config().hidden_size).cuda() + adapter.load_state_dict(torch.load(args.adapter, map_location="cuda")) + adapter.eval() + + items = [it for it in json.load(open(OUT / "star_data.json")) + if it["split"] == "test"][: args.n] + print(f"[{args.tag}] GSM carry-cot eval on {len(items)}, " + f"grid={args.grid} ff={args.feedforward}", flush=True) + + res = {"tag": args.tag, "grid": {}, "n": len(items)} + for cell in args.grid.split(","): + k, p = (int(x) for x in cell.split(":")) + t0 = time.time() + hits, per_label, per_item = 0, {}, [] + for i in range(0, len(items), args.batch): + chunk = items[i : i + args.batch] + enc = tok([chat_prompt(tok, it["question"], DIRECT_SUFFIX) + for it in chunk], return_tensors="pt", padding=True, + add_special_tokens=False).to("cuda") + if k == 0 and p == 0: + gen = model.generate(**enc, max_new_tokens=args.max_new, + do_sample=False) + else: + gen = generate_carry_c(looper, adapter, tok, + enc["input_ids"], + enc["attention_mask"], k, p, + max_new_tokens=args.max_new, + feedforward=args.feedforward) + for j, it in enumerate(chunk): + txt = tok.decode(gen[j, enc["input_ids"].shape[1]:], + skip_special_tokens=True) + ok = num_eq(last_number(txt), it["gold"]) + hits += ok + d = per_label.setdefault(it["label"], [0, 0]) + d[0] += ok + d[1] += 1 + per_item.append({"idx": it["idx"], "ok": bool(ok)}) + acc = hits / len(items) + by = {l: c / n for l, (c, n) in per_label.items()} + res["grid"][cell] = {"acc": acc, "by_label": by, + "per_item": per_item} + print(f"{cell}: acc={acc:.3f} " + f"by_label={ {l: round(v,3) for l,v in by.items()} }" + f" ({time.time()-t0:.0f}s)", flush=True) + json.dump(res, open(OUT / f"eval_{args.tag}.json", "w"), indent=1) + print("wrote", OUT / f"eval_{args.tag}.json") + + +if __name__ == "__main__": + main() diff --git a/scripts/jobs/zzz_l_gsm_e2a.sh b/scripts/jobs/zzz_l_gsm_e2a.sh new file mode 100644 index 0000000..14ebf04 --- /dev/null +++ b/scripts/jobs/zzz_l_gsm_e2a.sh @@ -0,0 +1,11 @@ +# gpuq-in: results-loop/star_data.json +# gpuq-out: results-loop/gsm_cot_data.json results-loop/eval_gsm_carrycot*.json results-loop/train_carrycot*_log.json +git pull origin main -q 2>/dev/null +P=/home/nils/jspace/.venv/bin/python +export JLENS_MODEL=google/gemma-4-E2B-it LOOP_OUT=/home/nils/jspace/results-loop +cd /home/nils/jspace/scripts +$P prep_gsm_cot.py +$P train_carry_cot.py +$P eval_carry_cot.py --adapter $LOOP_OUT/adapter_carrycot_e400.pt --tag gsm_carrycot_e400 --grid 0:0,2:2,2:6 --n 256 +$P train_carry_cot.py --feedforward +$P eval_carry_cot.py --adapter $LOOP_OUT/adapter_carrycot_ff_e400.pt --tag gsm_carrycot_ff_e400 --grid 2:2,2:6 --n 256 --feedforward diff --git a/scripts/path_gates.py b/scripts/path_gates.py new file mode 100644 index 0000000..03fa0a3 --- /dev/null +++ b/scripts/path_gates.py @@ -0,0 +1,170 @@ +"""Learnable soft path through ALL layers (pre-registration item 14). + +The frozen model = 35 same-typed functions on one residual bus. Instead of +a hand-fixed loop over L14-30, learn a gate matrix g[t, l] in [0,1]: on +loop iteration t, layer l's residual delta is scaled by g[t, l] (prompt +positions only; generated/answer positions always run ungated). Gates are +initialized as a Gaussian bump over depth centered mid-band, so at init a +loop iteration is approximately the hand band pass — then SGD may move the +compute envelope anywhere in [0, n_layers). The anchor merge adapter is +kept at each iteration boundary for stability (rho < 1). + +Reading the result: if the learned envelope concentrates on the lens band, +gradient descent independently rediscovers the workspace; if it wins with +mass elsewhere, the lens placement story needs revision. + +E2B caveat (stated in advance): KV sharing makes attention deltas of +layers >= 15 loop-inert; gate mass there is interpretable for MLP deltas +only. +""" + +import math + +import torch +import torch.nn as nn +from torch.utils.checkpoint import checkpoint + +from loop_common import BAND, BandLooper, _text_model + + +class PathGates(nn.Module): + """g[t, l] = sigmoid(logits[t, l]); row 0 = warm sweep, rows 1..k = loops.""" + + def __init__(self, n_layers, k_max, mu=None, sigma=6.0, band=BAND): + super().__init__() + mu = (band[0] + band[1]) / 2 if mu is None else mu + init = torch.empty(k_max + 1, n_layers) + for l in range(n_layers): + p = math.exp(-((l - mu) ** 2) / (2 * sigma ** 2)) + p = min(max(p, 1e-3), 1 - 1e-3) + init[:, l] = math.log(p / (1 - p)) + self.logits = nn.Parameter(init) + + def g(self, t): + return torch.sigmoid( + self.logits[min(t, self.logits.shape[0] - 1)].float()) + + def envelope(self): + with torch.no_grad(): + return torch.sigmoid(self.logits.float()).tolist() + + +class GatedLooper(BandLooper): + """BandLooper over the FULL depth with per-iteration per-layer gates.""" + + def __init__(self, model, gates): + tm = _text_model(model) + super().__init__(model, band=(0, len(tm.layers) - 1)) + self.gates = gates + + def _gated(self, h, calls, t, loop_mask=None): + g = self.gates.g(t).to(h.device) + x = h + for i in range(self.l0, self.l1 + 1): + args, kwargs = calls[i] + out = self.tm.layers[i](x, *args, **kwargs) + if isinstance(out, tuple): + out = out[0] + gi = g[i].to(x.dtype) + if loop_mask is not None: # ungated (g=1) off the prompt span + gi = torch.where(loop_mask[..., None], gi, + torch.ones_like(loop_mask[..., None], + dtype=x.dtype)) + x = x + gi * (out - x) + return x + + def loop_logits(self, adapter, input_ids, k, attention_mask=None, + use_checkpoint=False, return_states=False, + last_only=False, loop_mask=None, feedforward=False, + bptt=None): + calls, base_logits = self.capture(input_ids, attention_mask, + logits_to_keep=1 if last_only else 0) + if k == 0: + return (base_logits, None) if return_states else base_logits + del base_logits + e = self._hin[self.l0].detach() + + def sweep(x, t): + if use_checkpoint: + return checkpoint( + lambda x_: self._gated(x_, calls, t, loop_mask), x, + use_reentrant=False) + return self._gated(x, calls, t, loop_mask) + + s = sweep(e, 0) # warm sweep, t=0 (gates trainable here too) + states = [s] + n_nograd = max(0, k - bptt) if bptt else 0 + for i in range(k): + if i < n_nograd: + with torch.no_grad(): + x = adapter(e, s) + if loop_mask is not None: + x = torch.where(loop_mask[..., None], x, e) + s = self._gated(x, calls, i + 1, loop_mask) + s = s.detach() + states.append(s) + continue + x = adapter(e, s) + if loop_mask is not None: + x = torch.where(loop_mask[..., None], x, e) + s = sweep(x, i + 1) + states.append(s) + logits = self.suffix_logits(s, calls, last_only=last_only) + return (logits, states) if return_states else logits + + @torch.no_grad() + def generate_frozen_prompt(self, adapter, tok, input_ids, k, + max_new_tokens=220, attention_mask=None, + stop_strs=(), feedforward=False, + conv_out=None): + """Per-layer KV write-in: run the final gated sweep recording every + layer's INPUT, then one native prefill with pre-forward hooks + swapping each layer's hidden_states to the recorded stream — the + cache then holds exactly the gated states; decode is native.""" + if k == 0: + return super().generate_frozen_prompt( + adapter, tok, input_ids, 0, max_new_tokens=max_new_tokens, + attention_mask=attention_mask, stop_strs=stop_strs) + calls, _ = self.capture(input_ids, attention_mask, logits_to_keep=1) + e = self._hin[self.l0] + s = self._gated(e, calls, 0) + for i in range(k): + x = adapter(e, s) + if i < k - 1: + s = self._gated(x, calls, i + 1) + # final sweep: record per-layer inputs of the gated stream + xs = {} + g = self.gates.g(k).to(x.device) + h = x + for i in range(self.l0, self.l1 + 1): + xs[i] = h + args, kwargs = calls[i] + out = self.tm.layers[i](h, *args, **kwargs) + if isinstance(out, tuple): + out = out[0] + h = h + g[i].to(h.dtype) * (out - h) + del calls + + P = input_ids.shape[1] + handles = [] + for i in range(self.l0, self.l1 + 1): + def pre(mod, args, kwargs, i=i): + hh = kwargs.get("hidden_states", + args[0] if args else None) + if hh is not None and hh.shape[1] == P: # prefill only + if "hidden_states" in kwargs: + kwargs["hidden_states"] = xs[i].to(hh.dtype) + return args, kwargs + return (xs[i].to(hh.dtype),) + args[1:], kwargs + return None + handles.append(self.tm.layers[i].register_forward_pre_hook( + pre, with_kwargs=True)) + try: + gen = self.model.generate( + input_ids=input_ids, attention_mask=attention_mask, + max_new_tokens=max_new_tokens, do_sample=False, + pad_token_id=tok.pad_token_id or 0) + finally: + for hd in handles: + hd.remove() + return gen diff --git a/scripts/prep_gsm_cot.py b/scripts/prep_gsm_cot.py new file mode 100644 index 0000000..d9de0fa --- /dev/null +++ b/scripts/prep_gsm_cot.py @@ -0,0 +1,66 @@ +"""E2 stage A prep (item 21): harvest TERSE verified CoTs for GSM8K train. + +For each non-drop train item, sample a compressed scratchpad ("at most 3 +short steps"), keep it only if the final number matches gold (STaR +filter). Output: results-loop/gsm_cot_data.json rows +{idx, label, cot} — the dense supervision the latent-mode arms never had. +""" + +import json +import os +import sys +import time +from pathlib import Path + +import torch + +from loop_common import chat_prompt, last_number, num_eq + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from jlens.core import load_model # noqa: E402 + +OUT = Path(os.environ.get("LOOP_OUT", + Path(__file__).resolve().parent.parent / "results-loop")) +TERSE_SUFFIX = ("\nSolve in at most 3 short steps, one line each, digits " + "only (like '4*6=24'). Then give the last line exactly as " + "'Answer: N'.") +BATCH = 16 +MAX_NEW = 120 + + +@torch.no_grad() +def main(): + model, tok = load_model(dtype=torch.bfloat16) + tok.padding_side = "left" + items = [it for it in json.load(open(OUT / "star_data.json")) + if it["split"] == "train" and it["label"] != "drop"] + print(f"harvesting terse CoTs for {len(items)} train items", flush=True) + rows, kept = [], 0 + t0 = time.time() + for i in range(0, len(items), BATCH): + chunk = items[i : i + BATCH] + enc = tok([chat_prompt(tok, it["question"], TERSE_SUFFIX) + for it in chunk], return_tensors="pt", padding=True, + add_special_tokens=False).to("cuda") + gen = model.generate(**enc, max_new_tokens=MAX_NEW, do_sample=False) + for j, it in enumerate(chunk): + txt = tok.decode(gen[j, enc["input_ids"].shape[1]:], + skip_special_tokens=True).strip() + ok = num_eq(last_number(txt), it["gold"]) + if ok: + kept += 1 + rows.append({"idx": it["idx"], "label": it["label"], + "cot": txt}) + if i % 80 == 0: + print(f"[{i+len(chunk)}/{len(items)}] kept={kept} " + f"({time.time()-t0:.0f}s)", flush=True) + json.dump(rows, open(OUT / "gsm_cot_data.json", "w"), indent=1) + by = {} + for r in rows: + by[r["label"]] = by.get(r["label"], 0) + 1 + print(f"wrote {len(rows)} verified terse CoTs {by} -> gsm_cot_data.json", + flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/train_carry_cot.py b/scripts/train_carry_cot.py new file mode 100644 index 0000000..ff36ae0 --- /dev/null +++ b/scripts/train_carry_cot.py @@ -0,0 +1,159 @@ +"""E2 stage A (item 21): dense short-CoT supervision through the carry +whiteboard on GSM8K. + +train_carry.py skeleton, one change that matters: CE targets are the +model's own VERIFIED terse scratchpad + answer (gsm_cot_data.json, +~30-60 tokens) instead of the ~3-token bare answer — the dense-output +ingredient the latent GSM arms structurally lacked. +Control: --feedforward = same supervision, adapter(e,e), no recurrence. +""" + +import argparse +import json +import os +import math +import random +import sys +import time +from pathlib import Path + +import torch +import torch.nn.functional as F + +from carry_common import PAUSE_ID, carry_logits +from loop_common import BandLooper, MergeAdapter, chat_prompt, DIRECT_SUFFIX + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from jlens.core import load_model # noqa: E402 + +OUT = Path(os.environ.get("LOOP_OUT", + Path(__file__).resolve().parent.parent / "results-loop")) +STEPS = 600 +BATCH = 4 +LR = 1e-3 +WARMUP = 20 +K_PREFILL = 2 +P_BY_LABEL = {"easy": 2, "hard": 6} + +ap = argparse.ArgumentParser() +ap.add_argument("--feedforward", action="store_true") +ap.add_argument("--seed", type=int, default=0) +ARGS = ap.parse_args() +TAG = ("carrycot_ff" if ARGS.feedforward else "carrycot") + ( + f"_s{ARGS.seed}" if ARGS.seed else "") + + +def lr_at(step): + if step < WARMUP: + return LR * (step + 1) / WARMUP + t = (step - WARMUP) / max(1, STEPS - WARMUP) + return 1e-4 + 0.5 * (LR - 1e-4) * (1 + math.cos(math.pi * t)) + + +def build_batch(tok, items, p, device="cuda"): + seqs, labs, plens = [], [], [] + for it in items: + pr = tok(chat_prompt(tok, it["question"], DIRECT_SUFFIX), + add_special_tokens=False)["input_ids"] + a = tok(it["cot"] + "", + add_special_tokens=False)["input_ids"] + seqs.append(pr + [PAUSE_ID] * p + a) + labs.append([-100] * (len(pr) + p) + a) + plens.append(len(pr)) + T = max(len(s) for s in seqs) + pad = tok.pad_token_id or 0 + ids = torch.full((len(seqs), T), pad, dtype=torch.long) + lab = torch.full((len(seqs), T), -100, dtype=torch.long) + msk = torch.zeros((len(seqs), T), dtype=torch.long) + for i, (s, l) in enumerate(zip(seqs, labs)): + ids[i, : len(s)] = torch.tensor(s) + lab[i, : len(s)] = torch.tensor(l) + msk[i, : len(s)] = 1 + return (ids.to(device), msk.to(device), lab.to(device), + torch.tensor(plens, device=device)) + + +@torch.no_grad() +def val_loss(looper, adapter, tok, items, p): + tot, n = 0.0, 0 + for i in range(0, len(items), BATCH): + ids, msk, lab, plens = build_batch(tok, items[i : i + BATCH], p) + logits = carry_logits(looper, adapter, ids, msk, plens, K_PREFILL, + feedforward=ARGS.feedforward) + loss = F.cross_entropy(logits[:, :-1].flatten(0, 1).float(), + lab[:, 1:].flatten(), ignore_index=-100) + tot += loss.item() * len(ids) + n += len(ids) + return tot / n + + +def main(): + rng = random.Random(ARGS.seed) + torch.manual_seed(ARGS.seed) + star = {it["idx"]: it for it in json.load(open(OUT / "star_data.json")) + if it["split"] == "train"} + cots = json.load(open(OUT / "gsm_cot_data.json")) + data = [] + for r in cots: + it = star.get(r["idx"]) + if it is None: + continue + data.append({"question": it["question"], "label": r["label"], + "cot": r["cot"]}) + + model, tok = load_model(dtype=torch.bfloat16) + for pp in model.parameters(): + pp.requires_grad_(False) + looper = BandLooper(model) + adapter = MergeAdapter( + d=model.config.get_text_config().hidden_size).cuda() + opt = torch.optim.AdamW(adapter.parameters(), lr=LR, weight_decay=0.01) + + keep = [it for it in data + if len(tok(it["question"])["input_ids"]) + + len(tok(it["cot"])["input_ids"]) + 16 <= 460] + rng.shuffle(keep) + pool = {l: [it for it in keep if it["label"] == l] + for l in ("easy", "hard")} + val = {l: pool[l][:12] for l in pool} + pool = {l: pool[l][12:] for l in pool} + print(f"pool: easy={len(pool['easy'])} hard={len(pool['hard'])}", + flush=True) + if min(len(v) for v in pool.values()) < BATCH: + print("INSUFFICIENT POOL — aborting", flush=True) + return + + log = [] + t0 = time.time() + for step in range(STEPS): + lbl = ("easy", "hard")[step % 2] + batch = rng.sample(pool[lbl], BATCH) + p = P_BY_LABEL[lbl] + ids, msk, lab, plens = build_batch(tok, batch, p) + for g in opt.param_groups: + g["lr"] = lr_at(step) + logits = carry_logits(looper, adapter, ids, msk, plens, K_PREFILL, + use_checkpoint=True, + feedforward=ARGS.feedforward) + loss = F.cross_entropy(logits[:, :-1].flatten(0, 1).float(), + lab[:, 1:].flatten(), ignore_index=-100) + opt.zero_grad(set_to_none=True) + loss.backward() + torch.nn.utils.clip_grad_norm_(adapter.parameters(), 1.0) + opt.step() + log.append({"step": step, "loss": loss.item()}) + if step % 10 == 0: + print(f"step {step:4d} {lbl:4s} loss={loss.item():.4f} " + f"({(time.time()-t0)/(step+1):.1f}s/step)", flush=True) + if step % 200 == 199 or step == STEPS - 1: + for l in ("easy", "hard"): + v = val_loss(looper, adapter, tok, val[l], P_BY_LABEL[l]) + print(f" val@{step}: {l}={v:.3f}", flush=True) + torch.save(adapter.state_dict(), + OUT / f"adapter_{TAG}_e{step+1}.pt") + json.dump(log, open(OUT / f"train_{TAG}_log.json", "w")) + print("done", flush=True) + + +if __name__ == "__main__": + main()