diff --git a/PAPER.md b/PAPER.md index e24df21..a6bd700 100644 --- a/PAPER.md +++ b/PAPER.md @@ -22,10 +22,11 @@ pooled across MBPP, HumanEval, and Rust/MultiPL-E, the plan-dependent bucket moves from 4.2% to 35.6% (McNemar p≈1.5e-10) — and **placement is decisive, not a convenience**: the gain appears only when the loop enters at the lens-identified boundary (L14), collapsing at L13 and below and structurally -nulling above. Roughly a third of the gross effect is generic perturbation -(the untrained merge alone reaches ~18%; the bucket conditions on k=0 -failure, so regression-to-mean contributes to any intervention); the -loop-specific net effect is ~+20 points over that floor. +nulling above. A substantial part of the gross effect is generic +perturbation — the untrained merge alone reaches 20.0% on the same bucket +(it conditions on k=0 failure, so regression-to-mean contributes to any +intervention) — but the loop-specific net effect over that floor is real: +paired on items, loop-vs-untrained-merge is 17–4 discordant (p=0.007). A complete attribution program then deflates the mechanism's mystique: the loop's content is *amortizable* (plan-distillation into the same adapter, @@ -70,7 +71,9 @@ not laws.)* three benchmarks, 4.2%→35.6%, p≈1.5e-10. Overall accuracy is statistically unchanged on MBPP (p=0.34) and improved on HumanEval transfer (58.5%→66.5%, p=0.011). Net of the untrained-merge floor - (~18%), the loop-specific effect is ~+20 points. + (20.0% at n=55), the loop-specific effect is +17.5 points (seed mean) + and **survives the paired test** (loop vs untrained merge on hard, + p=0.007; distill vs untrained, p=0.0075). 3. **A deflationary mechanism finding.** The trained loop converges to a fixed point by k≈3–4 and behaves as *amortized plan content*, not @@ -191,8 +194,8 @@ MBPP hard bucket (plan-dependent, n=55 unless noted): | arm | hard pass@1 | overall | |---|---|---| | base (k=0, bit-exact) | 5.5% | 51.8% | -| untrained loop (α-merge only, n=28) | 17.9% | ~52% | -| trained FF, no recurrence (n=28) | 17.9% | ~52% | +| untrained loop (α-merge only, k=4) | 20.0% | 50.2% | +| trained FF, no recurrence (k=1) | 27.3% | 53.6% | | pause-16 registers (width) | 36.4% | 55.2% | | **trained loop k=4** (seed mean, 5 seeds) | **37.5±5.5** (best 43.6) | 53.6% | | rung-2: + entrance-faded band LoRA (n=28) | 42.9/46.4 (2 seeds) | 51.2/52.4 | @@ -209,11 +212,13 @@ ordering is not. **Net accounting.** The attribution-critical comparison is trained-loop vs *untrained merge*, not vs base: gross 5.5→37.5 (seed mean), of which the -untrained perturbation floor is ~18 points — the loop-specific net effect -is ~+20 points. The untrained-loop and trained-FF control rows above are -from the 250-item era (hard n=28); full-bucket (n=55) reruns of both -controls, enabling the paired loop-vs-untrained test, are running and will -replace these rows. [CONTROLS-N55 PENDING] +untrained perturbation floor is 20.0 points — the loop-specific net is ++17.5 (seed mean) / +23.6 (best seed), and the paired item-level test is +decisive (loop-only 17, untrained-only 4, p=0.007; distill likewise +p=0.0075). The trained-FF control (27.3%) sits between floor and loop, +not significantly above the floor (p=0.48): weights alone buy little +without either recurrence or plan supervision. All controls now n=500 / +hard n=55, same harness. ### 3.3 The decisive tests: nothing stacks diff --git a/results-loop/STATS.md b/results-loop/STATS.md index 6c8e9d8..7cc7ac6 100644 --- a/results-loop/STATS.md +++ b/results-loop/STATS.md @@ -2,6 +2,8 @@ ## Headline numbers (Wilson 95% CIs) +- **MBPP untrained merge k=4 (n=500 rerun)**: overall 0.502 [0.458, 0.546] (n=500); hard 0.200 [0.116, 0.324] (n=55) +- **MBPP trained FF k=1 (n=500 rerun)**: overall 0.536 [0.492, 0.579] (n=500); hard 0.273 [0.173, 0.402] (n=55) - **MBPP loop s0 k=0 (base)**: overall 0.518 [0.474, 0.561] (n=500); hard 0.055 [0.019, 0.149] (n=55) - **MBPP loop s0 k=2**: overall 0.530 [0.486, 0.573] (n=500); hard 0.309 [0.203, 0.440] (n=55) - **MBPP loop s0 k=4**: overall 0.536 [0.492, 0.579] (n=500); hard 0.436 [0.314, 0.567] (n=55) @@ -22,6 +24,10 @@ - loop k=4 vs k=0, overall: A-only 30, B-only 39, n=500, p=0.3356 (n.s.) - loop k=4 vs k=0, hard: A-only 1, B-only 22, n=55, p=5.722e-06 (**significant**) +- loop k=4 vs UNTRAINED merge k=4, hard (net effect): A-only 4, B-only 17, n=55, p=0.007197 (**significant**) +- loop k=4 vs UNTRAINED merge k=4, overall: A-only 29, B-only 46, n=500, p=0.06395 (n.s.) +- trained FF vs UNTRAINED merge, hard: A-only 7, B-only 11, n=55, p=0.4807 (n.s.) +- distill k=1 vs UNTRAINED merge k=4, hard: A-only 3, B-only 15, n=55, p=0.007538 (**significant**) - distill k=1 vs loop k=4, overall: A-only 33, B-only 37, n=500, p=0.7202 (n.s.) - distill k=1 vs loop k=4, hard: A-only 10, B-only 9, n=55, p=1 (n.s.) - stack-train k=4 vs distill k=1, hard: A-only 10, B-only 6, n=55, p=0.4545 (n.s.) diff --git a/results-loop/eval_code_code_ff_full.json b/results-loop/eval_code_code_ff_full.json new file mode 100644 index 0000000..820e487 --- /dev/null +++ b/results-loop/eval_code_code_ff_full.json @@ -0,0 +1,4026 @@ +{ + "tag": "code_ff_full", + "ks": { + "0": { + "acc": 0.524, + "by_label": { + "easy": 0.9884615384615385, + "hard": 0.03636363636363636, + "drop": 0.016216216216216217 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": false + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": true + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": true + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": true + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": false + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": true + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": false + }, + { + "task_id": 131, + "ok": false + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": false + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": false + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": false + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": false + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": true + }, + { + "task_id": 195, + "ok": false + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": false + }, + { + "task_id": 213, + "ok": true + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": false + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": false + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": true + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": true + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": false + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + }, + { + "task_id": 261, + "ok": true + }, + { + "task_id": 262, + "ok": true + }, + { + "task_id": 263, + "ok": true + }, + { + "task_id": 264, + "ok": false + }, + { + "task_id": 265, + "ok": true + }, + { + "task_id": 266, + "ok": true + }, + { + "task_id": 267, + "ok": true + }, + { + "task_id": 268, + "ok": false + }, + { + "task_id": 269, + "ok": true + }, + { + "task_id": 270, + "ok": false + }, + { + "task_id": 271, + "ok": true + }, + { + "task_id": 272, + "ok": true + }, + { + "task_id": 273, + "ok": true + }, + { + "task_id": 274, + "ok": true + }, + { + "task_id": 275, + "ok": false + }, + { + "task_id": 276, + "ok": false + }, + { + "task_id": 277, + "ok": false + }, + { + "task_id": 278, + "ok": true + }, + { + "task_id": 279, + "ok": false + }, + { + "task_id": 280, + "ok": true + }, + { + "task_id": 281, + "ok": true + }, + { + "task_id": 282, + "ok": false + }, + { + "task_id": 283, + "ok": true + }, + { + "task_id": 284, + "ok": true + }, + { + "task_id": 285, + "ok": true + }, + { + "task_id": 286, + "ok": false + }, + { + "task_id": 287, + "ok": true + }, + { + "task_id": 288, + "ok": false + }, + { + "task_id": 289, + "ok": false + }, + { + "task_id": 290, + "ok": true + }, + { + "task_id": 291, + "ok": false + }, + { + "task_id": 292, + "ok": true + }, + { + "task_id": 293, + "ok": true + }, + { + "task_id": 294, + "ok": true + }, + { + "task_id": 295, + "ok": false + }, + { + "task_id": 296, + "ok": true + }, + { + "task_id": 297, + "ok": true + }, + { + "task_id": 298, + "ok": false + }, + { + "task_id": 299, + "ok": true + }, + { + "task_id": 300, + "ok": false + }, + { + "task_id": 301, + "ok": true + }, + { + "task_id": 302, + "ok": false + }, + { + "task_id": 303, + "ok": false + }, + { + "task_id": 304, + "ok": false + }, + { + "task_id": 305, + "ok": false + }, + { + "task_id": 306, + "ok": false + }, + { + "task_id": 307, + "ok": false + }, + { + "task_id": 308, + "ok": true + }, + { + "task_id": 309, + "ok": true + }, + { + "task_id": 310, + "ok": false + }, + { + "task_id": 311, + "ok": false + }, + { + "task_id": 312, + "ok": false + }, + { + "task_id": 313, + "ok": false + }, + { + "task_id": 314, + "ok": false + }, + { + "task_id": 315, + "ok": true + }, + { + "task_id": 316, + "ok": true + }, + { + "task_id": 317, + "ok": false + }, + { + "task_id": 318, + "ok": false + }, + { + "task_id": 319, + "ok": false + }, + { + "task_id": 320, + "ok": false + }, + { + "task_id": 321, + "ok": false + }, + { + "task_id": 322, + "ok": true + }, + { + "task_id": 323, + "ok": false + }, + { + "task_id": 324, + "ok": false + }, + { + "task_id": 325, + "ok": false + }, + { + "task_id": 326, + "ok": false + }, + { + "task_id": 327, + "ok": true + }, + { + "task_id": 328, + "ok": false + }, + { + "task_id": 329, + "ok": true + }, + { + "task_id": 330, + "ok": true + }, + { + "task_id": 331, + "ok": false + }, + { + "task_id": 332, + "ok": true + }, + { + "task_id": 333, + "ok": true + }, + { + "task_id": 334, + "ok": true + }, + { + "task_id": 335, + "ok": false + }, + { + "task_id": 336, + "ok": true + }, + { + "task_id": 337, + "ok": false + }, + { + "task_id": 338, + "ok": true + }, + { + "task_id": 339, + "ok": false + }, + { + "task_id": 340, + "ok": true + }, + { + "task_id": 341, + "ok": true + }, + { + "task_id": 342, + "ok": false + }, + { + "task_id": 343, + "ok": true + }, + { + "task_id": 344, + "ok": true + }, + { + "task_id": 345, + "ok": true + }, + { + "task_id": 346, + "ok": false + }, + { + "task_id": 347, + "ok": true + }, + { + "task_id": 348, + "ok": false + }, + { + "task_id": 349, + "ok": true + }, + { + "task_id": 350, + "ok": true + }, + { + "task_id": 351, + "ok": false + }, + { + "task_id": 352, + "ok": true + }, + { + "task_id": 353, + "ok": true + }, + { + "task_id": 354, + "ok": true + }, + { + "task_id": 355, + "ok": false + }, + { + "task_id": 356, + "ok": true + }, + { + "task_id": 357, + "ok": true + }, + { + "task_id": 358, + "ok": true + }, + { + "task_id": 359, + "ok": false + }, + { + "task_id": 360, + "ok": false + }, + { + "task_id": 361, + "ok": false + }, + { + "task_id": 362, + "ok": false + }, + { + "task_id": 363, + "ok": true + }, + { + "task_id": 364, + "ok": false + }, + { + "task_id": 365, + "ok": true + }, + { + "task_id": 366, + "ok": true + }, + { + "task_id": 367, + "ok": false + }, + { + "task_id": 368, + "ok": true + }, + { + "task_id": 369, + "ok": false + }, + { + "task_id": 370, + "ok": false + }, + { + "task_id": 371, + "ok": false + }, + { + "task_id": 372, + "ok": true + }, + { + "task_id": 373, + "ok": true + }, + { + "task_id": 374, + "ok": false + }, + { + "task_id": 375, + "ok": false + }, + { + "task_id": 376, + "ok": false + }, + { + "task_id": 377, + "ok": true + }, + { + "task_id": 378, + "ok": true + }, + { + "task_id": 379, + "ok": true + }, + { + "task_id": 380, + "ok": false + }, + { + "task_id": 381, + "ok": true + }, + { + "task_id": 382, + "ok": false + }, + { + "task_id": 383, + "ok": false + }, + { + "task_id": 384, + "ok": false + }, + { + "task_id": 385, + "ok": true + }, + { + "task_id": 386, + "ok": false + }, + { + "task_id": 387, + "ok": true + }, + { + "task_id": 388, + "ok": true + }, + { + "task_id": 389, + "ok": true + }, + { + "task_id": 390, + "ok": false + }, + { + "task_id": 391, + "ok": true + }, + { + "task_id": 392, + "ok": false + }, + { + "task_id": 393, + "ok": false + }, + { + "task_id": 394, + "ok": true + }, + { + "task_id": 395, + "ok": true + }, + { + "task_id": 396, + "ok": false + }, + { + "task_id": 397, + "ok": true + }, + { + "task_id": 398, + "ok": false + }, + { + "task_id": 399, + "ok": true + }, + { + "task_id": 400, + "ok": false + }, + { + "task_id": 401, + "ok": true + }, + { + "task_id": 402, + "ok": false + }, + { + "task_id": 403, + "ok": false + }, + { + "task_id": 404, + "ok": true + }, + { + "task_id": 405, + "ok": true + }, + { + "task_id": 406, + "ok": true + }, + { + "task_id": 407, + "ok": false + }, + { + "task_id": 408, + "ok": false + }, + { + "task_id": 409, + "ok": true + }, + { + "task_id": 410, + "ok": false + }, + { + "task_id": 411, + "ok": true + }, + { + "task_id": 412, + "ok": true + }, + { + "task_id": 413, + "ok": true + }, + { + "task_id": 414, + "ok": true + }, + { + "task_id": 415, + "ok": false + }, + { + "task_id": 416, + "ok": false + }, + { + "task_id": 417, + "ok": false + }, + { + "task_id": 418, + "ok": true + }, + { + "task_id": 419, + "ok": true + }, + { + "task_id": 420, + "ok": true + }, + { + "task_id": 421, + "ok": true + }, + { + "task_id": 422, + "ok": true + }, + { + "task_id": 423, + "ok": false + }, + { + "task_id": 424, + "ok": true + }, + { + "task_id": 425, + "ok": true + }, + { + "task_id": 426, + "ok": true + }, + { + "task_id": 427, + "ok": true + }, + { + "task_id": 428, + "ok": true + }, + { + "task_id": 429, + "ok": true + }, + { + "task_id": 430, + "ok": false + }, + { + "task_id": 431, + "ok": true + }, + { + "task_id": 432, + "ok": false + }, + { + "task_id": 433, + "ok": true + }, + { + "task_id": 434, + "ok": true + }, + { + "task_id": 435, + "ok": true + }, + { + "task_id": 436, + "ok": false + }, + { + "task_id": 437, + "ok": false + }, + { + "task_id": 438, + "ok": false + }, + { + "task_id": 439, + "ok": true + }, + { + "task_id": 440, + "ok": false + }, + { + "task_id": 441, + "ok": true + }, + { + "task_id": 442, + "ok": false + }, + { + "task_id": 443, + "ok": false + }, + { + "task_id": 444, + "ok": false + }, + { + "task_id": 445, + "ok": true + }, + { + "task_id": 446, + "ok": true + }, + { + "task_id": 447, + "ok": true + }, + { + "task_id": 448, + "ok": false + }, + { + "task_id": 449, + "ok": false + }, + { + "task_id": 450, + "ok": true + }, + { + "task_id": 451, + "ok": true + }, + { + "task_id": 452, + "ok": true + }, + { + "task_id": 453, + "ok": true + }, + { + "task_id": 454, + "ok": true + }, + { + "task_id": 455, + "ok": true + }, + { + "task_id": 456, + "ok": true + }, + { + "task_id": 457, + "ok": true + }, + { + "task_id": 458, + "ok": true + }, + { + "task_id": 459, + "ok": true + }, + { + "task_id": 460, + "ok": true + }, + { + "task_id": 461, + "ok": false + }, + { + "task_id": 462, + "ok": false + }, + { + "task_id": 463, + "ok": false + }, + { + "task_id": 464, + "ok": true + }, + { + "task_id": 465, + "ok": true + }, + { + "task_id": 466, + "ok": true + }, + { + "task_id": 467, + "ok": false + }, + { + "task_id": 468, + "ok": false + }, + { + "task_id": 469, + "ok": false + }, + { + "task_id": 470, + "ok": true + }, + { + "task_id": 471, + "ok": true + }, + { + "task_id": 472, + "ok": false + }, + { + "task_id": 473, + "ok": false + }, + { + "task_id": 474, + "ok": true + }, + { + "task_id": 475, + "ok": true + }, + { + "task_id": 476, + "ok": true + }, + { + "task_id": 477, + "ok": true + }, + { + "task_id": 478, + "ok": true + }, + { + "task_id": 479, + "ok": true + }, + { + "task_id": 480, + "ok": false + }, + { + "task_id": 481, + "ok": false + }, + { + "task_id": 482, + "ok": true + }, + { + "task_id": 483, + "ok": false + }, + { + "task_id": 484, + "ok": true + }, + { + "task_id": 485, + "ok": true + }, + { + "task_id": 486, + "ok": false + }, + { + "task_id": 487, + "ok": true + }, + { + "task_id": 488, + "ok": false + }, + { + "task_id": 489, + "ok": false + }, + { + "task_id": 490, + "ok": false + }, + { + "task_id": 491, + "ok": true + }, + { + "task_id": 492, + "ok": true + }, + { + "task_id": 493, + "ok": false + }, + { + "task_id": 494, + "ok": false + }, + { + "task_id": 495, + "ok": true + }, + { + "task_id": 496, + "ok": false + }, + { + "task_id": 497, + "ok": false + }, + { + "task_id": 498, + "ok": true + }, + { + "task_id": 499, + "ok": true + }, + { + "task_id": 500, + "ok": false + }, + { + "task_id": 501, + "ok": false + }, + { + "task_id": 502, + "ok": true + }, + { + "task_id": 503, + "ok": false + }, + { + "task_id": 504, + "ok": true + }, + { + "task_id": 505, + "ok": true + }, + { + "task_id": 506, + "ok": true + }, + { + "task_id": 507, + "ok": true + }, + { + "task_id": 508, + "ok": false + }, + { + "task_id": 509, + "ok": true + }, + { + "task_id": 510, + "ok": true + } + ] + }, + "1": { + "acc": 0.536, + "by_label": { + "easy": 0.9461538461538461, + "hard": 0.2727272727272727, + "drop": 0.03783783783783784 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": true + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": true + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": true + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": true + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": true + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": true + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": true + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": true + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": true + }, + { + "task_id": 131, + "ok": true + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": false + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": false + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": false + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": true + }, + { + "task_id": 166, + "ok": false + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": false + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": true + }, + { + "task_id": 195, + "ok": false + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": false + }, + { + "task_id": 213, + "ok": false + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": true + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": false + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": false + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": true + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": true + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + }, + { + "task_id": 261, + "ok": true + }, + { + "task_id": 262, + "ok": true + }, + { + "task_id": 263, + "ok": true + }, + { + "task_id": 264, + "ok": false + }, + { + "task_id": 265, + "ok": true + }, + { + "task_id": 266, + "ok": true + }, + { + "task_id": 267, + "ok": true + }, + { + "task_id": 268, + "ok": false + }, + { + "task_id": 269, + "ok": true + }, + { + "task_id": 270, + "ok": false + }, + { + "task_id": 271, + "ok": true + }, + { + "task_id": 272, + "ok": true + }, + { + "task_id": 273, + "ok": true + }, + { + "task_id": 274, + "ok": true + }, + { + "task_id": 275, + "ok": false + }, + { + "task_id": 276, + "ok": false + }, + { + "task_id": 277, + "ok": false + }, + { + "task_id": 278, + "ok": true + }, + { + "task_id": 279, + "ok": false + }, + { + "task_id": 280, + "ok": true + }, + { + "task_id": 281, + "ok": true + }, + { + "task_id": 282, + "ok": false + }, + { + "task_id": 283, + "ok": true + }, + { + "task_id": 284, + "ok": true + }, + { + "task_id": 285, + "ok": true + }, + { + "task_id": 286, + "ok": false + }, + { + "task_id": 287, + "ok": true + }, + { + "task_id": 288, + "ok": false + }, + { + "task_id": 289, + "ok": false + }, + { + "task_id": 290, + "ok": true + }, + { + "task_id": 291, + "ok": false + }, + { + "task_id": 292, + "ok": true + }, + { + "task_id": 293, + "ok": true + }, + { + "task_id": 294, + "ok": true + }, + { + "task_id": 295, + "ok": false + }, + { + "task_id": 296, + "ok": false + }, + { + "task_id": 297, + "ok": true + }, + { + "task_id": 298, + "ok": false + }, + { + "task_id": 299, + "ok": false + }, + { + "task_id": 300, + "ok": false + }, + { + "task_id": 301, + "ok": false + }, + { + "task_id": 302, + "ok": false + }, + { + "task_id": 303, + "ok": false + }, + { + "task_id": 304, + "ok": false + }, + { + "task_id": 305, + "ok": false + }, + { + "task_id": 306, + "ok": false + }, + { + "task_id": 307, + "ok": false + }, + { + "task_id": 308, + "ok": true + }, + { + "task_id": 309, + "ok": true + }, + { + "task_id": 310, + "ok": false + }, + { + "task_id": 311, + "ok": false + }, + { + "task_id": 312, + "ok": false + }, + { + "task_id": 313, + "ok": false + }, + { + "task_id": 314, + "ok": false + }, + { + "task_id": 315, + "ok": false + }, + { + "task_id": 316, + "ok": true + }, + { + "task_id": 317, + "ok": false + }, + { + "task_id": 318, + "ok": false + }, + { + "task_id": 319, + "ok": true + }, + { + "task_id": 320, + "ok": false + }, + { + "task_id": 321, + "ok": false + }, + { + "task_id": 322, + "ok": true + }, + { + "task_id": 323, + "ok": false + }, + { + "task_id": 324, + "ok": false + }, + { + "task_id": 325, + "ok": false + }, + { + "task_id": 326, + "ok": false + }, + { + "task_id": 327, + "ok": true + }, + { + "task_id": 328, + "ok": false + }, + { + "task_id": 329, + "ok": true + }, + { + "task_id": 330, + "ok": true + }, + { + "task_id": 331, + "ok": false + }, + { + "task_id": 332, + "ok": true + }, + { + "task_id": 333, + "ok": true + }, + { + "task_id": 334, + "ok": true + }, + { + "task_id": 335, + "ok": false + }, + { + "task_id": 336, + "ok": true + }, + { + "task_id": 337, + "ok": false + }, + { + "task_id": 338, + "ok": true + }, + { + "task_id": 339, + "ok": false + }, + { + "task_id": 340, + "ok": true + }, + { + "task_id": 341, + "ok": true + }, + { + "task_id": 342, + "ok": false + }, + { + "task_id": 343, + "ok": true + }, + { + "task_id": 344, + "ok": false + }, + { + "task_id": 345, + "ok": true + }, + { + "task_id": 346, + "ok": false + }, + { + "task_id": 347, + "ok": true + }, + { + "task_id": 348, + "ok": false + }, + { + "task_id": 349, + "ok": true + }, + { + "task_id": 350, + "ok": true + }, + { + "task_id": 351, + "ok": false + }, + { + "task_id": 352, + "ok": true + }, + { + "task_id": 353, + "ok": true + }, + { + "task_id": 354, + "ok": true + }, + { + "task_id": 355, + "ok": false + }, + { + "task_id": 356, + "ok": true + }, + { + "task_id": 357, + "ok": true + }, + { + "task_id": 358, + "ok": true + }, + { + "task_id": 359, + "ok": false + }, + { + "task_id": 360, + "ok": false + }, + { + "task_id": 361, + "ok": false + }, + { + "task_id": 362, + "ok": false + }, + { + "task_id": 363, + "ok": true + }, + { + "task_id": 364, + "ok": true + }, + { + "task_id": 365, + "ok": true + }, + { + "task_id": 366, + "ok": true + }, + { + "task_id": 367, + "ok": false + }, + { + "task_id": 368, + "ok": true + }, + { + "task_id": 369, + "ok": true + }, + { + "task_id": 370, + "ok": false + }, + { + "task_id": 371, + "ok": false + }, + { + "task_id": 372, + "ok": true + }, + { + "task_id": 373, + "ok": true + }, + { + "task_id": 374, + "ok": false + }, + { + "task_id": 375, + "ok": false + }, + { + "task_id": 376, + "ok": false + }, + { + "task_id": 377, + "ok": true + }, + { + "task_id": 378, + "ok": true + }, + { + "task_id": 379, + "ok": true + }, + { + "task_id": 380, + "ok": false + }, + { + "task_id": 381, + "ok": true + }, + { + "task_id": 382, + "ok": false + }, + { + "task_id": 383, + "ok": false + }, + { + "task_id": 384, + "ok": false + }, + { + "task_id": 385, + "ok": true + }, + { + "task_id": 386, + "ok": false + }, + { + "task_id": 387, + "ok": true + }, + { + "task_id": 388, + "ok": true + }, + { + "task_id": 389, + "ok": true + }, + { + "task_id": 390, + "ok": false + }, + { + "task_id": 391, + "ok": true + }, + { + "task_id": 392, + "ok": true + }, + { + "task_id": 393, + "ok": true + }, + { + "task_id": 394, + "ok": true + }, + { + "task_id": 395, + "ok": true + }, + { + "task_id": 396, + "ok": false + }, + { + "task_id": 397, + "ok": true + }, + { + "task_id": 398, + "ok": false + }, + { + "task_id": 399, + "ok": true + }, + { + "task_id": 400, + "ok": false + }, + { + "task_id": 401, + "ok": true + }, + { + "task_id": 402, + "ok": false + }, + { + "task_id": 403, + "ok": true + }, + { + "task_id": 404, + "ok": true + }, + { + "task_id": 405, + "ok": true + }, + { + "task_id": 406, + "ok": true + }, + { + "task_id": 407, + "ok": false + }, + { + "task_id": 408, + "ok": false + }, + { + "task_id": 409, + "ok": false + }, + { + "task_id": 410, + "ok": false + }, + { + "task_id": 411, + "ok": false + }, + { + "task_id": 412, + "ok": true + }, + { + "task_id": 413, + "ok": true + }, + { + "task_id": 414, + "ok": true + }, + { + "task_id": 415, + "ok": false + }, + { + "task_id": 416, + "ok": false + }, + { + "task_id": 417, + "ok": false + }, + { + "task_id": 418, + "ok": true + }, + { + "task_id": 419, + "ok": true + }, + { + "task_id": 420, + "ok": true + }, + { + "task_id": 421, + "ok": true + }, + { + "task_id": 422, + "ok": true + }, + { + "task_id": 423, + "ok": false + }, + { + "task_id": 424, + "ok": true + }, + { + "task_id": 425, + "ok": true + }, + { + "task_id": 426, + "ok": true + }, + { + "task_id": 427, + "ok": true + }, + { + "task_id": 428, + "ok": true + }, + { + "task_id": 429, + "ok": true + }, + { + "task_id": 430, + "ok": false + }, + { + "task_id": 431, + "ok": true + }, + { + "task_id": 432, + "ok": false + }, + { + "task_id": 433, + "ok": false + }, + { + "task_id": 434, + "ok": true + }, + { + "task_id": 435, + "ok": true + }, + { + "task_id": 436, + "ok": false + }, + { + "task_id": 437, + "ok": false + }, + { + "task_id": 438, + "ok": false + }, + { + "task_id": 439, + "ok": true + }, + { + "task_id": 440, + "ok": false + }, + { + "task_id": 441, + "ok": true + }, + { + "task_id": 442, + "ok": false + }, + { + "task_id": 443, + "ok": false + }, + { + "task_id": 444, + "ok": false + }, + { + "task_id": 445, + "ok": true + }, + { + "task_id": 446, + "ok": true + }, + { + "task_id": 447, + "ok": true + }, + { + "task_id": 448, + "ok": false + }, + { + "task_id": 449, + "ok": false + }, + { + "task_id": 450, + "ok": true + }, + { + "task_id": 451, + "ok": true + }, + { + "task_id": 452, + "ok": false + }, + { + "task_id": 453, + "ok": true + }, + { + "task_id": 454, + "ok": true + }, + { + "task_id": 455, + "ok": true + }, + { + "task_id": 456, + "ok": true + }, + { + "task_id": 457, + "ok": true + }, + { + "task_id": 458, + "ok": true + }, + { + "task_id": 459, + "ok": true + }, + { + "task_id": 460, + "ok": true + }, + { + "task_id": 461, + "ok": false + }, + { + "task_id": 462, + "ok": false + }, + { + "task_id": 463, + "ok": false + }, + { + "task_id": 464, + "ok": true + }, + { + "task_id": 465, + "ok": true + }, + { + "task_id": 466, + "ok": true + }, + { + "task_id": 467, + "ok": false + }, + { + "task_id": 468, + "ok": false + }, + { + "task_id": 469, + "ok": false + }, + { + "task_id": 470, + "ok": true + }, + { + "task_id": 471, + "ok": true + }, + { + "task_id": 472, + "ok": false + }, + { + "task_id": 473, + "ok": false + }, + { + "task_id": 474, + "ok": true + }, + { + "task_id": 475, + "ok": true + }, + { + "task_id": 476, + "ok": true + }, + { + "task_id": 477, + "ok": true + }, + { + "task_id": 478, + "ok": true + }, + { + "task_id": 479, + "ok": true + }, + { + "task_id": 480, + "ok": true + }, + { + "task_id": 481, + "ok": false + }, + { + "task_id": 482, + "ok": true + }, + { + "task_id": 483, + "ok": false + }, + { + "task_id": 484, + "ok": true + }, + { + "task_id": 485, + "ok": true + }, + { + "task_id": 486, + "ok": false + }, + { + "task_id": 487, + "ok": true + }, + { + "task_id": 488, + "ok": false + }, + { + "task_id": 489, + "ok": false + }, + { + "task_id": 490, + "ok": false + }, + { + "task_id": 491, + "ok": true + }, + { + "task_id": 492, + "ok": true + }, + { + "task_id": 493, + "ok": false + }, + { + "task_id": 494, + "ok": false + }, + { + "task_id": 495, + "ok": true + }, + { + "task_id": 496, + "ok": true + }, + { + "task_id": 497, + "ok": false + }, + { + "task_id": 498, + "ok": true + }, + { + "task_id": 499, + "ok": true + }, + { + "task_id": 500, + "ok": false + }, + { + "task_id": 501, + "ok": false + }, + { + "task_id": 502, + "ok": true + }, + { + "task_id": 503, + "ok": false + }, + { + "task_id": 504, + "ok": true + }, + { + "task_id": 505, + "ok": true + }, + { + "task_id": 506, + "ok": true + }, + { + "task_id": 507, + "ok": true + }, + { + "task_id": 508, + "ok": true + }, + { + "task_id": 509, + "ok": true + }, + { + "task_id": 510, + "ok": true + } + ] + } + }, + "n": 500 +} \ No newline at end of file diff --git a/results-loop/eval_code_untrained_full.json b/results-loop/eval_code_untrained_full.json new file mode 100644 index 0000000..f5dbb44 --- /dev/null +++ b/results-loop/eval_code_untrained_full.json @@ -0,0 +1,6036 @@ +{ + "tag": "untrained_full", + "ks": { + "0": { + "acc": 0.524, + "by_label": { + "easy": 0.9884615384615385, + "hard": 0.03636363636363636, + "drop": 0.016216216216216217 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": false + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": true + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": true + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": true + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": false + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": true + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": false + }, + { + "task_id": 131, + "ok": false + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": false + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": false + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": false + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": false + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": true + }, + { + "task_id": 195, + "ok": false + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": false + }, + { + "task_id": 213, + "ok": true + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": false + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": false + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": true + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": true + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": false + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + }, + { + "task_id": 261, + "ok": true + }, + { + "task_id": 262, + "ok": true + }, + { + "task_id": 263, + "ok": true + }, + { + "task_id": 264, + "ok": false + }, + { + "task_id": 265, + "ok": true + }, + { + "task_id": 266, + "ok": true + }, + { + "task_id": 267, + "ok": true + }, + { + "task_id": 268, + "ok": false + }, + { + "task_id": 269, + "ok": true + }, + { + "task_id": 270, + "ok": false + }, + { + "task_id": 271, + "ok": true + }, + { + "task_id": 272, + "ok": true + }, + { + "task_id": 273, + "ok": true + }, + { + "task_id": 274, + "ok": true + }, + { + "task_id": 275, + "ok": false + }, + { + "task_id": 276, + "ok": false + }, + { + "task_id": 277, + "ok": false + }, + { + "task_id": 278, + "ok": true + }, + { + "task_id": 279, + "ok": false + }, + { + "task_id": 280, + "ok": true + }, + { + "task_id": 281, + "ok": true + }, + { + "task_id": 282, + "ok": false + }, + { + "task_id": 283, + "ok": true + }, + { + "task_id": 284, + "ok": true + }, + { + "task_id": 285, + "ok": true + }, + { + "task_id": 286, + "ok": false + }, + { + "task_id": 287, + "ok": true + }, + { + "task_id": 288, + "ok": false + }, + { + "task_id": 289, + "ok": false + }, + { + "task_id": 290, + "ok": true + }, + { + "task_id": 291, + "ok": false + }, + { + "task_id": 292, + "ok": true + }, + { + "task_id": 293, + "ok": true + }, + { + "task_id": 294, + "ok": true + }, + { + "task_id": 295, + "ok": false + }, + { + "task_id": 296, + "ok": true + }, + { + "task_id": 297, + "ok": true + }, + { + "task_id": 298, + "ok": false + }, + { + "task_id": 299, + "ok": true + }, + { + "task_id": 300, + "ok": false + }, + { + "task_id": 301, + "ok": true + }, + { + "task_id": 302, + "ok": false + }, + { + "task_id": 303, + "ok": false + }, + { + "task_id": 304, + "ok": false + }, + { + "task_id": 305, + "ok": false + }, + { + "task_id": 306, + "ok": false + }, + { + "task_id": 307, + "ok": false + }, + { + "task_id": 308, + "ok": true + }, + { + "task_id": 309, + "ok": true + }, + { + "task_id": 310, + "ok": false + }, + { + "task_id": 311, + "ok": false + }, + { + "task_id": 312, + "ok": false + }, + { + "task_id": 313, + "ok": false + }, + { + "task_id": 314, + "ok": false + }, + { + "task_id": 315, + "ok": true + }, + { + "task_id": 316, + "ok": true + }, + { + "task_id": 317, + "ok": false + }, + { + "task_id": 318, + "ok": false + }, + { + "task_id": 319, + "ok": false + }, + { + "task_id": 320, + "ok": false + }, + { + "task_id": 321, + "ok": false + }, + { + "task_id": 322, + "ok": true + }, + { + "task_id": 323, + "ok": false + }, + { + "task_id": 324, + "ok": false + }, + { + "task_id": 325, + "ok": false + }, + { + "task_id": 326, + "ok": false + }, + { + "task_id": 327, + "ok": true + }, + { + "task_id": 328, + "ok": false + }, + { + "task_id": 329, + "ok": true + }, + { + "task_id": 330, + "ok": true + }, + { + "task_id": 331, + "ok": false + }, + { + "task_id": 332, + "ok": true + }, + { + "task_id": 333, + "ok": true + }, + { + "task_id": 334, + "ok": true + }, + { + "task_id": 335, + "ok": false + }, + { + "task_id": 336, + "ok": true + }, + { + "task_id": 337, + "ok": false + }, + { + "task_id": 338, + "ok": true + }, + { + "task_id": 339, + "ok": false + }, + { + "task_id": 340, + "ok": true + }, + { + "task_id": 341, + "ok": true + }, + { + "task_id": 342, + "ok": false + }, + { + "task_id": 343, + "ok": true + }, + { + "task_id": 344, + "ok": true + }, + { + "task_id": 345, + "ok": true + }, + { + "task_id": 346, + "ok": false + }, + { + "task_id": 347, + "ok": true + }, + { + "task_id": 348, + "ok": false + }, + { + "task_id": 349, + "ok": true + }, + { + "task_id": 350, + "ok": true + }, + { + "task_id": 351, + "ok": false + }, + { + "task_id": 352, + "ok": true + }, + { + "task_id": 353, + "ok": true + }, + { + "task_id": 354, + "ok": true + }, + { + "task_id": 355, + "ok": false + }, + { + "task_id": 356, + "ok": true + }, + { + "task_id": 357, + "ok": true + }, + { + "task_id": 358, + "ok": true + }, + { + "task_id": 359, + "ok": false + }, + { + "task_id": 360, + "ok": false + }, + { + "task_id": 361, + "ok": false + }, + { + "task_id": 362, + "ok": false + }, + { + "task_id": 363, + "ok": true + }, + { + "task_id": 364, + "ok": false + }, + { + "task_id": 365, + "ok": true + }, + { + "task_id": 366, + "ok": true + }, + { + "task_id": 367, + "ok": false + }, + { + "task_id": 368, + "ok": true + }, + { + "task_id": 369, + "ok": false + }, + { + "task_id": 370, + "ok": false + }, + { + "task_id": 371, + "ok": false + }, + { + "task_id": 372, + "ok": true + }, + { + "task_id": 373, + "ok": true + }, + { + "task_id": 374, + "ok": false + }, + { + "task_id": 375, + "ok": false + }, + { + "task_id": 376, + "ok": false + }, + { + "task_id": 377, + "ok": true + }, + { + "task_id": 378, + "ok": true + }, + { + "task_id": 379, + "ok": true + }, + { + "task_id": 380, + "ok": false + }, + { + "task_id": 381, + "ok": true + }, + { + "task_id": 382, + "ok": false + }, + { + "task_id": 383, + "ok": false + }, + { + "task_id": 384, + "ok": false + }, + { + "task_id": 385, + "ok": true + }, + { + "task_id": 386, + "ok": false + }, + { + "task_id": 387, + "ok": true + }, + { + "task_id": 388, + "ok": true + }, + { + "task_id": 389, + "ok": true + }, + { + "task_id": 390, + "ok": false + }, + { + "task_id": 391, + "ok": true + }, + { + "task_id": 392, + "ok": false + }, + { + "task_id": 393, + "ok": false + }, + { + "task_id": 394, + "ok": true + }, + { + "task_id": 395, + "ok": true + }, + { + "task_id": 396, + "ok": false + }, + { + "task_id": 397, + "ok": true + }, + { + "task_id": 398, + "ok": false + }, + { + "task_id": 399, + "ok": true + }, + { + "task_id": 400, + "ok": false + }, + { + "task_id": 401, + "ok": true + }, + { + "task_id": 402, + "ok": false + }, + { + "task_id": 403, + "ok": false + }, + { + "task_id": 404, + "ok": true + }, + { + "task_id": 405, + "ok": true + }, + { + "task_id": 406, + "ok": true + }, + { + "task_id": 407, + "ok": false + }, + { + "task_id": 408, + "ok": false + }, + { + "task_id": 409, + "ok": true + }, + { + "task_id": 410, + "ok": false + }, + { + "task_id": 411, + "ok": true + }, + { + "task_id": 412, + "ok": true + }, + { + "task_id": 413, + "ok": true + }, + { + "task_id": 414, + "ok": true + }, + { + "task_id": 415, + "ok": false + }, + { + "task_id": 416, + "ok": false + }, + { + "task_id": 417, + "ok": false + }, + { + "task_id": 418, + "ok": true + }, + { + "task_id": 419, + "ok": true + }, + { + "task_id": 420, + "ok": true + }, + { + "task_id": 421, + "ok": true + }, + { + "task_id": 422, + "ok": true + }, + { + "task_id": 423, + "ok": false + }, + { + "task_id": 424, + "ok": true + }, + { + "task_id": 425, + "ok": true + }, + { + "task_id": 426, + "ok": true + }, + { + "task_id": 427, + "ok": true + }, + { + "task_id": 428, + "ok": true + }, + { + "task_id": 429, + "ok": true + }, + { + "task_id": 430, + "ok": false + }, + { + "task_id": 431, + "ok": true + }, + { + "task_id": 432, + "ok": false + }, + { + "task_id": 433, + "ok": true + }, + { + "task_id": 434, + "ok": true + }, + { + "task_id": 435, + "ok": true + }, + { + "task_id": 436, + "ok": false + }, + { + "task_id": 437, + "ok": false + }, + { + "task_id": 438, + "ok": false + }, + { + "task_id": 439, + "ok": true + }, + { + "task_id": 440, + "ok": false + }, + { + "task_id": 441, + "ok": true + }, + { + "task_id": 442, + "ok": false + }, + { + "task_id": 443, + "ok": false + }, + { + "task_id": 444, + "ok": false + }, + { + "task_id": 445, + "ok": true + }, + { + "task_id": 446, + "ok": true + }, + { + "task_id": 447, + "ok": true + }, + { + "task_id": 448, + "ok": false + }, + { + "task_id": 449, + "ok": false + }, + { + "task_id": 450, + "ok": true + }, + { + "task_id": 451, + "ok": true + }, + { + "task_id": 452, + "ok": true + }, + { + "task_id": 453, + "ok": true + }, + { + "task_id": 454, + "ok": true + }, + { + "task_id": 455, + "ok": true + }, + { + "task_id": 456, + "ok": true + }, + { + "task_id": 457, + "ok": true + }, + { + "task_id": 458, + "ok": true + }, + { + "task_id": 459, + "ok": true + }, + { + "task_id": 460, + "ok": true + }, + { + "task_id": 461, + "ok": false + }, + { + "task_id": 462, + "ok": false + }, + { + "task_id": 463, + "ok": false + }, + { + "task_id": 464, + "ok": true + }, + { + "task_id": 465, + "ok": true + }, + { + "task_id": 466, + "ok": true + }, + { + "task_id": 467, + "ok": false + }, + { + "task_id": 468, + "ok": false + }, + { + "task_id": 469, + "ok": false + }, + { + "task_id": 470, + "ok": true + }, + { + "task_id": 471, + "ok": true + }, + { + "task_id": 472, + "ok": false + }, + { + "task_id": 473, + "ok": false + }, + { + "task_id": 474, + "ok": true + }, + { + "task_id": 475, + "ok": true + }, + { + "task_id": 476, + "ok": true + }, + { + "task_id": 477, + "ok": true + }, + { + "task_id": 478, + "ok": true + }, + { + "task_id": 479, + "ok": true + }, + { + "task_id": 480, + "ok": false + }, + { + "task_id": 481, + "ok": false + }, + { + "task_id": 482, + "ok": true + }, + { + "task_id": 483, + "ok": false + }, + { + "task_id": 484, + "ok": true + }, + { + "task_id": 485, + "ok": true + }, + { + "task_id": 486, + "ok": false + }, + { + "task_id": 487, + "ok": true + }, + { + "task_id": 488, + "ok": false + }, + { + "task_id": 489, + "ok": false + }, + { + "task_id": 490, + "ok": false + }, + { + "task_id": 491, + "ok": true + }, + { + "task_id": 492, + "ok": true + }, + { + "task_id": 493, + "ok": false + }, + { + "task_id": 494, + "ok": false + }, + { + "task_id": 495, + "ok": true + }, + { + "task_id": 496, + "ok": false + }, + { + "task_id": 497, + "ok": false + }, + { + "task_id": 498, + "ok": true + }, + { + "task_id": 499, + "ok": true + }, + { + "task_id": 500, + "ok": false + }, + { + "task_id": 501, + "ok": false + }, + { + "task_id": 502, + "ok": true + }, + { + "task_id": 503, + "ok": false + }, + { + "task_id": 504, + "ok": true + }, + { + "task_id": 505, + "ok": true + }, + { + "task_id": 506, + "ok": true + }, + { + "task_id": 507, + "ok": true + }, + { + "task_id": 508, + "ok": false + }, + { + "task_id": 509, + "ok": true + }, + { + "task_id": 510, + "ok": true + } + ] + }, + "2": { + "acc": 0.492, + "by_label": { + "easy": 0.8692307692307693, + "hard": 0.18181818181818182, + "drop": 0.05405405405405406 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": true + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": false + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": false + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": false + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": false + }, + { + "task_id": 58, + "ok": false + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": false + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": false + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": true + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": false + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": false + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": false + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": true + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": true + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": false + }, + { + "task_id": 131, + "ok": false + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": true + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": true + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": false + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": true + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": true + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": false + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": true + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": false + }, + { + "task_id": 195, + "ok": false + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": true + }, + { + "task_id": 213, + "ok": true + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": false + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": false + }, + { + "task_id": 225, + "ok": false + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": true + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": false + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": false + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": true + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": true + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + }, + { + "task_id": 261, + "ok": true + }, + { + "task_id": 262, + "ok": true + }, + { + "task_id": 263, + "ok": true + }, + { + "task_id": 264, + "ok": false + }, + { + "task_id": 265, + "ok": false + }, + { + "task_id": 266, + "ok": true + }, + { + "task_id": 267, + "ok": true + }, + { + "task_id": 268, + "ok": false + }, + { + "task_id": 269, + "ok": true + }, + { + "task_id": 270, + "ok": true + }, + { + "task_id": 271, + "ok": true + }, + { + "task_id": 272, + "ok": false + }, + { + "task_id": 273, + "ok": true + }, + { + "task_id": 274, + "ok": false + }, + { + "task_id": 275, + "ok": false + }, + { + "task_id": 276, + "ok": false + }, + { + "task_id": 277, + "ok": false + }, + { + "task_id": 278, + "ok": true + }, + { + "task_id": 279, + "ok": false + }, + { + "task_id": 280, + "ok": true + }, + { + "task_id": 281, + "ok": true + }, + { + "task_id": 282, + "ok": true + }, + { + "task_id": 283, + "ok": true + }, + { + "task_id": 284, + "ok": true + }, + { + "task_id": 285, + "ok": true + }, + { + "task_id": 286, + "ok": false + }, + { + "task_id": 287, + "ok": true + }, + { + "task_id": 288, + "ok": false + }, + { + "task_id": 289, + "ok": false + }, + { + "task_id": 290, + "ok": true + }, + { + "task_id": 291, + "ok": false + }, + { + "task_id": 292, + "ok": true + }, + { + "task_id": 293, + "ok": true + }, + { + "task_id": 294, + "ok": true + }, + { + "task_id": 295, + "ok": false + }, + { + "task_id": 296, + "ok": false + }, + { + "task_id": 297, + "ok": true + }, + { + "task_id": 298, + "ok": false + }, + { + "task_id": 299, + "ok": false + }, + { + "task_id": 300, + "ok": false + }, + { + "task_id": 301, + "ok": false + }, + { + "task_id": 302, + "ok": false + }, + { + "task_id": 303, + "ok": false + }, + { + "task_id": 304, + "ok": false + }, + { + "task_id": 305, + "ok": false + }, + { + "task_id": 306, + "ok": false + }, + { + "task_id": 307, + "ok": false + }, + { + "task_id": 308, + "ok": true + }, + { + "task_id": 309, + "ok": true + }, + { + "task_id": 310, + "ok": false + }, + { + "task_id": 311, + "ok": false + }, + { + "task_id": 312, + "ok": false + }, + { + "task_id": 313, + "ok": false + }, + { + "task_id": 314, + "ok": false + }, + { + "task_id": 315, + "ok": false + }, + { + "task_id": 316, + "ok": true + }, + { + "task_id": 317, + "ok": true + }, + { + "task_id": 318, + "ok": false + }, + { + "task_id": 319, + "ok": true + }, + { + "task_id": 320, + "ok": false + }, + { + "task_id": 321, + "ok": false + }, + { + "task_id": 322, + "ok": true + }, + { + "task_id": 323, + "ok": false + }, + { + "task_id": 324, + "ok": false + }, + { + "task_id": 325, + "ok": true + }, + { + "task_id": 326, + "ok": false + }, + { + "task_id": 327, + "ok": true + }, + { + "task_id": 328, + "ok": false + }, + { + "task_id": 329, + "ok": true + }, + { + "task_id": 330, + "ok": true + }, + { + "task_id": 331, + "ok": false + }, + { + "task_id": 332, + "ok": true + }, + { + "task_id": 333, + "ok": true + }, + { + "task_id": 334, + "ok": true + }, + { + "task_id": 335, + "ok": false + }, + { + "task_id": 336, + "ok": true + }, + { + "task_id": 337, + "ok": false + }, + { + "task_id": 338, + "ok": true + }, + { + "task_id": 339, + "ok": false + }, + { + "task_id": 340, + "ok": true + }, + { + "task_id": 341, + "ok": true + }, + { + "task_id": 342, + "ok": false + }, + { + "task_id": 343, + "ok": false + }, + { + "task_id": 344, + "ok": false + }, + { + "task_id": 345, + "ok": true + }, + { + "task_id": 346, + "ok": false + }, + { + "task_id": 347, + "ok": true + }, + { + "task_id": 348, + "ok": false + }, + { + "task_id": 349, + "ok": true + }, + { + "task_id": 350, + "ok": true + }, + { + "task_id": 351, + "ok": false + }, + { + "task_id": 352, + "ok": true + }, + { + "task_id": 353, + "ok": true + }, + { + "task_id": 354, + "ok": false + }, + { + "task_id": 355, + "ok": false + }, + { + "task_id": 356, + "ok": true + }, + { + "task_id": 357, + "ok": true + }, + { + "task_id": 358, + "ok": true + }, + { + "task_id": 359, + "ok": false + }, + { + "task_id": 360, + "ok": false + }, + { + "task_id": 361, + "ok": true + }, + { + "task_id": 362, + "ok": false + }, + { + "task_id": 363, + "ok": true + }, + { + "task_id": 364, + "ok": false + }, + { + "task_id": 365, + "ok": true + }, + { + "task_id": 366, + "ok": true + }, + { + "task_id": 367, + "ok": false + }, + { + "task_id": 368, + "ok": false + }, + { + "task_id": 369, + "ok": false + }, + { + "task_id": 370, + "ok": true + }, + { + "task_id": 371, + "ok": false + }, + { + "task_id": 372, + "ok": true + }, + { + "task_id": 373, + "ok": true + }, + { + "task_id": 374, + "ok": false + }, + { + "task_id": 375, + "ok": false + }, + { + "task_id": 376, + "ok": false + }, + { + "task_id": 377, + "ok": true + }, + { + "task_id": 378, + "ok": true + }, + { + "task_id": 379, + "ok": true + }, + { + "task_id": 380, + "ok": false + }, + { + "task_id": 381, + "ok": true + }, + { + "task_id": 382, + "ok": false + }, + { + "task_id": 383, + "ok": false + }, + { + "task_id": 384, + "ok": false + }, + { + "task_id": 385, + "ok": true + }, + { + "task_id": 386, + "ok": false + }, + { + "task_id": 387, + "ok": false + }, + { + "task_id": 388, + "ok": true + }, + { + "task_id": 389, + "ok": true + }, + { + "task_id": 390, + "ok": false + }, + { + "task_id": 391, + "ok": true + }, + { + "task_id": 392, + "ok": false + }, + { + "task_id": 393, + "ok": false + }, + { + "task_id": 394, + "ok": true + }, + { + "task_id": 395, + "ok": true + }, + { + "task_id": 396, + "ok": false + }, + { + "task_id": 397, + "ok": true + }, + { + "task_id": 398, + "ok": true + }, + { + "task_id": 399, + "ok": true + }, + { + "task_id": 400, + "ok": false + }, + { + "task_id": 401, + "ok": true + }, + { + "task_id": 402, + "ok": false + }, + { + "task_id": 403, + "ok": false + }, + { + "task_id": 404, + "ok": true + }, + { + "task_id": 405, + "ok": true + }, + { + "task_id": 406, + "ok": true + }, + { + "task_id": 407, + "ok": false + }, + { + "task_id": 408, + "ok": false + }, + { + "task_id": 409, + "ok": true + }, + { + "task_id": 410, + "ok": false + }, + { + "task_id": 411, + "ok": true + }, + { + "task_id": 412, + "ok": true + }, + { + "task_id": 413, + "ok": true + }, + { + "task_id": 414, + "ok": true + }, + { + "task_id": 415, + "ok": false + }, + { + "task_id": 416, + "ok": false + }, + { + "task_id": 417, + "ok": false + }, + { + "task_id": 418, + "ok": true + }, + { + "task_id": 419, + "ok": true + }, + { + "task_id": 420, + "ok": true + }, + { + "task_id": 421, + "ok": true + }, + { + "task_id": 422, + "ok": true + }, + { + "task_id": 423, + "ok": false + }, + { + "task_id": 424, + "ok": true + }, + { + "task_id": 425, + "ok": true + }, + { + "task_id": 426, + "ok": true + }, + { + "task_id": 427, + "ok": true + }, + { + "task_id": 428, + "ok": true + }, + { + "task_id": 429, + "ok": true + }, + { + "task_id": 430, + "ok": false + }, + { + "task_id": 431, + "ok": true + }, + { + "task_id": 432, + "ok": false + }, + { + "task_id": 433, + "ok": false + }, + { + "task_id": 434, + "ok": false + }, + { + "task_id": 435, + "ok": true + }, + { + "task_id": 436, + "ok": false + }, + { + "task_id": 437, + "ok": false + }, + { + "task_id": 438, + "ok": false + }, + { + "task_id": 439, + "ok": true + }, + { + "task_id": 440, + "ok": false + }, + { + "task_id": 441, + "ok": true + }, + { + "task_id": 442, + "ok": false + }, + { + "task_id": 443, + "ok": false + }, + { + "task_id": 444, + "ok": false + }, + { + "task_id": 445, + "ok": true + }, + { + "task_id": 446, + "ok": true + }, + { + "task_id": 447, + "ok": true + }, + { + "task_id": 448, + "ok": false + }, + { + "task_id": 449, + "ok": false + }, + { + "task_id": 450, + "ok": true + }, + { + "task_id": 451, + "ok": true + }, + { + "task_id": 452, + "ok": false + }, + { + "task_id": 453, + "ok": true + }, + { + "task_id": 454, + "ok": true + }, + { + "task_id": 455, + "ok": true + }, + { + "task_id": 456, + "ok": true + }, + { + "task_id": 457, + "ok": true + }, + { + "task_id": 458, + "ok": true + }, + { + "task_id": 459, + "ok": true + }, + { + "task_id": 460, + "ok": true + }, + { + "task_id": 461, + "ok": false + }, + { + "task_id": 462, + "ok": false + }, + { + "task_id": 463, + "ok": false + }, + { + "task_id": 464, + "ok": false + }, + { + "task_id": 465, + "ok": true + }, + { + "task_id": 466, + "ok": true + }, + { + "task_id": 467, + "ok": false + }, + { + "task_id": 468, + "ok": false + }, + { + "task_id": 469, + "ok": false + }, + { + "task_id": 470, + "ok": true + }, + { + "task_id": 471, + "ok": false + }, + { + "task_id": 472, + "ok": false + }, + { + "task_id": 473, + "ok": false + }, + { + "task_id": 474, + "ok": true + }, + { + "task_id": 475, + "ok": true + }, + { + "task_id": 476, + "ok": true + }, + { + "task_id": 477, + "ok": true + }, + { + "task_id": 478, + "ok": true + }, + { + "task_id": 479, + "ok": true + }, + { + "task_id": 480, + "ok": false + }, + { + "task_id": 481, + "ok": false + }, + { + "task_id": 482, + "ok": true + }, + { + "task_id": 483, + "ok": false + }, + { + "task_id": 484, + "ok": true + }, + { + "task_id": 485, + "ok": true + }, + { + "task_id": 486, + "ok": false + }, + { + "task_id": 487, + "ok": true + }, + { + "task_id": 488, + "ok": false + }, + { + "task_id": 489, + "ok": false + }, + { + "task_id": 490, + "ok": false + }, + { + "task_id": 491, + "ok": false + }, + { + "task_id": 492, + "ok": true + }, + { + "task_id": 493, + "ok": false + }, + { + "task_id": 494, + "ok": false + }, + { + "task_id": 495, + "ok": true + }, + { + "task_id": 496, + "ok": false + }, + { + "task_id": 497, + "ok": false + }, + { + "task_id": 498, + "ok": true + }, + { + "task_id": 499, + "ok": true + }, + { + "task_id": 500, + "ok": false + }, + { + "task_id": 501, + "ok": false + }, + { + "task_id": 502, + "ok": true + }, + { + "task_id": 503, + "ok": false + }, + { + "task_id": 504, + "ok": true + }, + { + "task_id": 505, + "ok": true + }, + { + "task_id": 506, + "ok": true + }, + { + "task_id": 507, + "ok": true + }, + { + "task_id": 508, + "ok": false + }, + { + "task_id": 509, + "ok": true + }, + { + "task_id": 510, + "ok": false + } + ] + }, + "4": { + "acc": 0.502, + "by_label": { + "easy": 0.8769230769230769, + "hard": 0.2, + "drop": 0.06486486486486487 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": true + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": false + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": false + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": false + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": false + }, + { + "task_id": 58, + "ok": false + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": false + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": false + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": false + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": false + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": false + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": true + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": true + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": true + }, + { + "task_id": 131, + "ok": true + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": true + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": true + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": false + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": false + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": true + }, + { + "task_id": 166, + "ok": true + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": true + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": false + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": true + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": false + }, + { + "task_id": 195, + "ok": false + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": true + }, + { + "task_id": 213, + "ok": true + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": false + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": false + }, + { + "task_id": 225, + "ok": false + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": true + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": false + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": false + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": true + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": true + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": true + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + }, + { + "task_id": 261, + "ok": true + }, + { + "task_id": 262, + "ok": true + }, + { + "task_id": 263, + "ok": false + }, + { + "task_id": 264, + "ok": false + }, + { + "task_id": 265, + "ok": false + }, + { + "task_id": 266, + "ok": true + }, + { + "task_id": 267, + "ok": true + }, + { + "task_id": 268, + "ok": false + }, + { + "task_id": 269, + "ok": true + }, + { + "task_id": 270, + "ok": true + }, + { + "task_id": 271, + "ok": true + }, + { + "task_id": 272, + "ok": true + }, + { + "task_id": 273, + "ok": true + }, + { + "task_id": 274, + "ok": false + }, + { + "task_id": 275, + "ok": false + }, + { + "task_id": 276, + "ok": false + }, + { + "task_id": 277, + "ok": false + }, + { + "task_id": 278, + "ok": true + }, + { + "task_id": 279, + "ok": false + }, + { + "task_id": 280, + "ok": true + }, + { + "task_id": 281, + "ok": true + }, + { + "task_id": 282, + "ok": false + }, + { + "task_id": 283, + "ok": true + }, + { + "task_id": 284, + "ok": true + }, + { + "task_id": 285, + "ok": true + }, + { + "task_id": 286, + "ok": false + }, + { + "task_id": 287, + "ok": true + }, + { + "task_id": 288, + "ok": false + }, + { + "task_id": 289, + "ok": false + }, + { + "task_id": 290, + "ok": true + }, + { + "task_id": 291, + "ok": false + }, + { + "task_id": 292, + "ok": true + }, + { + "task_id": 293, + "ok": true + }, + { + "task_id": 294, + "ok": false + }, + { + "task_id": 295, + "ok": false + }, + { + "task_id": 296, + "ok": false + }, + { + "task_id": 297, + "ok": true + }, + { + "task_id": 298, + "ok": true + }, + { + "task_id": 299, + "ok": true + }, + { + "task_id": 300, + "ok": false + }, + { + "task_id": 301, + "ok": false + }, + { + "task_id": 302, + "ok": true + }, + { + "task_id": 303, + "ok": false + }, + { + "task_id": 304, + "ok": false + }, + { + "task_id": 305, + "ok": false + }, + { + "task_id": 306, + "ok": false + }, + { + "task_id": 307, + "ok": false + }, + { + "task_id": 308, + "ok": false + }, + { + "task_id": 309, + "ok": true + }, + { + "task_id": 310, + "ok": false + }, + { + "task_id": 311, + "ok": false + }, + { + "task_id": 312, + "ok": false + }, + { + "task_id": 313, + "ok": false + }, + { + "task_id": 314, + "ok": false + }, + { + "task_id": 315, + "ok": true + }, + { + "task_id": 316, + "ok": true + }, + { + "task_id": 317, + "ok": true + }, + { + "task_id": 318, + "ok": false + }, + { + "task_id": 319, + "ok": true + }, + { + "task_id": 320, + "ok": false + }, + { + "task_id": 321, + "ok": false + }, + { + "task_id": 322, + "ok": true + }, + { + "task_id": 323, + "ok": false + }, + { + "task_id": 324, + "ok": false + }, + { + "task_id": 325, + "ok": true + }, + { + "task_id": 326, + "ok": false + }, + { + "task_id": 327, + "ok": true + }, + { + "task_id": 328, + "ok": false + }, + { + "task_id": 329, + "ok": true + }, + { + "task_id": 330, + "ok": true + }, + { + "task_id": 331, + "ok": false + }, + { + "task_id": 332, + "ok": true + }, + { + "task_id": 333, + "ok": true + }, + { + "task_id": 334, + "ok": true + }, + { + "task_id": 335, + "ok": false + }, + { + "task_id": 336, + "ok": true + }, + { + "task_id": 337, + "ok": false + }, + { + "task_id": 338, + "ok": true + }, + { + "task_id": 339, + "ok": false + }, + { + "task_id": 340, + "ok": true + }, + { + "task_id": 341, + "ok": true + }, + { + "task_id": 342, + "ok": false + }, + { + "task_id": 343, + "ok": true + }, + { + "task_id": 344, + "ok": false + }, + { + "task_id": 345, + "ok": true + }, + { + "task_id": 346, + "ok": false + }, + { + "task_id": 347, + "ok": true + }, + { + "task_id": 348, + "ok": false + }, + { + "task_id": 349, + "ok": true + }, + { + "task_id": 350, + "ok": false + }, + { + "task_id": 351, + "ok": false + }, + { + "task_id": 352, + "ok": true + }, + { + "task_id": 353, + "ok": true + }, + { + "task_id": 354, + "ok": false + }, + { + "task_id": 355, + "ok": false + }, + { + "task_id": 356, + "ok": true + }, + { + "task_id": 357, + "ok": true + }, + { + "task_id": 358, + "ok": true + }, + { + "task_id": 359, + "ok": false + }, + { + "task_id": 360, + "ok": false + }, + { + "task_id": 361, + "ok": true + }, + { + "task_id": 362, + "ok": false + }, + { + "task_id": 363, + "ok": true + }, + { + "task_id": 364, + "ok": false + }, + { + "task_id": 365, + "ok": true + }, + { + "task_id": 366, + "ok": true + }, + { + "task_id": 367, + "ok": false + }, + { + "task_id": 368, + "ok": false + }, + { + "task_id": 369, + "ok": false + }, + { + "task_id": 370, + "ok": false + }, + { + "task_id": 371, + "ok": false + }, + { + "task_id": 372, + "ok": true + }, + { + "task_id": 373, + "ok": true + }, + { + "task_id": 374, + "ok": false + }, + { + "task_id": 375, + "ok": false + }, + { + "task_id": 376, + "ok": false + }, + { + "task_id": 377, + "ok": true + }, + { + "task_id": 378, + "ok": true + }, + { + "task_id": 379, + "ok": true + }, + { + "task_id": 380, + "ok": false + }, + { + "task_id": 381, + "ok": true + }, + { + "task_id": 382, + "ok": false + }, + { + "task_id": 383, + "ok": false + }, + { + "task_id": 384, + "ok": false + }, + { + "task_id": 385, + "ok": true + }, + { + "task_id": 386, + "ok": false + }, + { + "task_id": 387, + "ok": false + }, + { + "task_id": 388, + "ok": true + }, + { + "task_id": 389, + "ok": true + }, + { + "task_id": 390, + "ok": false + }, + { + "task_id": 391, + "ok": true + }, + { + "task_id": 392, + "ok": false + }, + { + "task_id": 393, + "ok": false + }, + { + "task_id": 394, + "ok": true + }, + { + "task_id": 395, + "ok": true + }, + { + "task_id": 396, + "ok": false + }, + { + "task_id": 397, + "ok": true + }, + { + "task_id": 398, + "ok": true + }, + { + "task_id": 399, + "ok": true + }, + { + "task_id": 400, + "ok": false + }, + { + "task_id": 401, + "ok": true + }, + { + "task_id": 402, + "ok": false + }, + { + "task_id": 403, + "ok": false + }, + { + "task_id": 404, + "ok": true + }, + { + "task_id": 405, + "ok": true + }, + { + "task_id": 406, + "ok": true + }, + { + "task_id": 407, + "ok": false + }, + { + "task_id": 408, + "ok": false + }, + { + "task_id": 409, + "ok": true + }, + { + "task_id": 410, + "ok": false + }, + { + "task_id": 411, + "ok": true + }, + { + "task_id": 412, + "ok": true + }, + { + "task_id": 413, + "ok": true + }, + { + "task_id": 414, + "ok": true + }, + { + "task_id": 415, + "ok": false + }, + { + "task_id": 416, + "ok": false + }, + { + "task_id": 417, + "ok": false + }, + { + "task_id": 418, + "ok": true + }, + { + "task_id": 419, + "ok": true + }, + { + "task_id": 420, + "ok": true + }, + { + "task_id": 421, + "ok": true + }, + { + "task_id": 422, + "ok": true + }, + { + "task_id": 423, + "ok": false + }, + { + "task_id": 424, + "ok": true + }, + { + "task_id": 425, + "ok": true + }, + { + "task_id": 426, + "ok": true + }, + { + "task_id": 427, + "ok": true + }, + { + "task_id": 428, + "ok": true + }, + { + "task_id": 429, + "ok": true + }, + { + "task_id": 430, + "ok": false + }, + { + "task_id": 431, + "ok": true + }, + { + "task_id": 432, + "ok": false + }, + { + "task_id": 433, + "ok": false + }, + { + "task_id": 434, + "ok": true + }, + { + "task_id": 435, + "ok": true + }, + { + "task_id": 436, + "ok": false + }, + { + "task_id": 437, + "ok": false + }, + { + "task_id": 438, + "ok": false + }, + { + "task_id": 439, + "ok": true + }, + { + "task_id": 440, + "ok": false + }, + { + "task_id": 441, + "ok": true + }, + { + "task_id": 442, + "ok": false + }, + { + "task_id": 443, + "ok": false + }, + { + "task_id": 444, + "ok": false + }, + { + "task_id": 445, + "ok": true + }, + { + "task_id": 446, + "ok": true + }, + { + "task_id": 447, + "ok": true + }, + { + "task_id": 448, + "ok": false + }, + { + "task_id": 449, + "ok": false + }, + { + "task_id": 450, + "ok": true + }, + { + "task_id": 451, + "ok": true + }, + { + "task_id": 452, + "ok": false + }, + { + "task_id": 453, + "ok": true + }, + { + "task_id": 454, + "ok": true + }, + { + "task_id": 455, + "ok": true + }, + { + "task_id": 456, + "ok": true + }, + { + "task_id": 457, + "ok": true + }, + { + "task_id": 458, + "ok": true + }, + { + "task_id": 459, + "ok": true + }, + { + "task_id": 460, + "ok": true + }, + { + "task_id": 461, + "ok": false + }, + { + "task_id": 462, + "ok": false + }, + { + "task_id": 463, + "ok": false + }, + { + "task_id": 464, + "ok": false + }, + { + "task_id": 465, + "ok": true + }, + { + "task_id": 466, + "ok": true + }, + { + "task_id": 467, + "ok": false + }, + { + "task_id": 468, + "ok": false + }, + { + "task_id": 469, + "ok": false + }, + { + "task_id": 470, + "ok": true + }, + { + "task_id": 471, + "ok": false + }, + { + "task_id": 472, + "ok": false + }, + { + "task_id": 473, + "ok": false + }, + { + "task_id": 474, + "ok": true + }, + { + "task_id": 475, + "ok": true + }, + { + "task_id": 476, + "ok": true + }, + { + "task_id": 477, + "ok": true + }, + { + "task_id": 478, + "ok": true + }, + { + "task_id": 479, + "ok": true + }, + { + "task_id": 480, + "ok": false + }, + { + "task_id": 481, + "ok": false + }, + { + "task_id": 482, + "ok": true + }, + { + "task_id": 483, + "ok": false + }, + { + "task_id": 484, + "ok": true + }, + { + "task_id": 485, + "ok": true + }, + { + "task_id": 486, + "ok": false + }, + { + "task_id": 487, + "ok": true + }, + { + "task_id": 488, + "ok": false + }, + { + "task_id": 489, + "ok": false + }, + { + "task_id": 490, + "ok": false + }, + { + "task_id": 491, + "ok": false + }, + { + "task_id": 492, + "ok": true + }, + { + "task_id": 493, + "ok": false + }, + { + "task_id": 494, + "ok": false + }, + { + "task_id": 495, + "ok": true + }, + { + "task_id": 496, + "ok": true + }, + { + "task_id": 497, + "ok": false + }, + { + "task_id": 498, + "ok": true + }, + { + "task_id": 499, + "ok": true + }, + { + "task_id": 500, + "ok": false + }, + { + "task_id": 501, + "ok": false + }, + { + "task_id": 502, + "ok": true + }, + { + "task_id": 503, + "ok": false + }, + { + "task_id": 504, + "ok": true + }, + { + "task_id": 505, + "ok": true + }, + { + "task_id": 506, + "ok": true + }, + { + "task_id": 507, + "ok": true + }, + { + "task_id": 508, + "ok": false + }, + { + "task_id": 509, + "ok": true + }, + { + "task_id": 510, + "ok": false + } + ] + } + }, + "n": 500 +} \ No newline at end of file diff --git a/results-loop/fig_ladder.png b/results-loop/fig_ladder.png index 660a706..fbd7f52 100644 Binary files a/results-loop/fig_ladder.png and b/results-loop/fig_ladder.png differ diff --git a/results-loop/stats_final.json b/results-loop/stats_final.json index 5a4513c..7170d94 100644 --- a/results-loop/stats_final.json +++ b/results-loop/stats_final.json @@ -1,4 +1,40 @@ { + "MBPP untrained merge k=4 (n=500 rerun)": { + "overall": { + "acc": 0.502, + "n": 500, + "ci": [ + 0.458, + 0.546 + ] + }, + "hard": { + "acc": 0.2, + "n": 55, + "ci": [ + 0.116, + 0.324 + ] + } + }, + "MBPP trained FF k=1 (n=500 rerun)": { + "overall": { + "acc": 0.536, + "n": 500, + "ci": [ + 0.492, + 0.579 + ] + }, + "hard": { + "acc": 0.2727272727272727, + "n": 55, + "ci": [ + 0.173, + 0.402 + ] + } + }, "MBPP loop s0 k=0 (base)": { "overall": { "acc": 0.518, @@ -275,6 +311,30 @@ "b_only": 22, "p": 5.7220458984375e-06 }, + "loop k=4 vs UNTRAINED merge k=4, hard (net effect)": { + "n": 55, + "a_only": 4, + "b_only": 17, + "p": 0.007197380065917969 + }, + "loop k=4 vs UNTRAINED merge k=4, overall": { + "n": 500, + "a_only": 29, + "b_only": 46, + "p": 0.06394991646706696 + }, + "trained FF vs UNTRAINED merge, hard": { + "n": 55, + "a_only": 7, + "b_only": 11, + "p": 0.480682373046875 + }, + "distill k=1 vs UNTRAINED merge k=4, hard": { + "n": 55, + "a_only": 3, + "b_only": 15, + "p": 0.007537841796875 + }, "distill k=1 vs loop k=4, overall": { "n": 500, "a_only": 33, diff --git a/scripts/figures_final.py b/scripts/figures_final.py index 69b4a13..2920b11 100644 --- a/scripts/figures_final.py +++ b/scripts/figures_final.py @@ -100,13 +100,10 @@ def fig_ladder(): _, h0 = ev(OUT / "eval_code_code_s0_full.json", 0) add("base (k=0, exact)", h0, NH, GRAY) - # untrained E2B control (250-item era, n_hard=28) - d = json.load(open(OUT / "eval_code_untrained.json")) - hu = max(v["by_label"]["hard"] for k, v in d["ks"].items() if k != "0") - add("untrained loop (best k)", hu, 28, GRAY) - d = json.load(open(OUT / "eval_code_ff.json")) - add("trained FF (no recurrence)", d["ks"]["1"]["by_label"]["hard"], 28, - ORANGE) + _, hu = ev(OUT / "eval_code_untrained_full.json", 4) + add("untrained loop (perturbation floor)", hu, NH, GRAY) + _, hf = ev(OUT / "eval_code_code_ff_full.json", 1) + add("trained FF (no recurrence)", hf, NH, ORANGE) _, hp = ev(OUT / "eval_code_pause16.json", 1) add("pause-16 registers (width)", hp, NH, ORANGE) _, hl = ev(OUT / "eval_code_code_s0_full.json", 4) diff --git a/scripts/stats_final.py b/scripts/stats_final.py index 56833f6..d77930c 100644 --- a/scripts/stats_final.py +++ b/scripts/stats_final.py @@ -85,6 +85,10 @@ def main(): lines += ["## Headline numbers (Wilson 95% CIs)", ""] ARMS = [ # (label, file, k, hard-subset) + ("MBPP untrained merge k=4 (n=500 rerun)", + "eval_code_untrained_full.json", 4, mbpp_hard), + ("MBPP trained FF k=1 (n=500 rerun)", + "eval_code_code_ff_full.json", 1, mbpp_hard), ("MBPP loop s0 k=0 (base)", "eval_code_code_s0_full.json", 0, mbpp_hard), ("MBPP loop s0 k=2", "eval_code_code_s0_full.json", 2, mbpp_hard), ("MBPP loop s0 k=4", "eval_code_code_s0_full.json", 4, mbpp_hard), @@ -173,9 +177,16 @@ def main(): dist1, _ = arm_maps["MBPP distill s1 k=1 (FF)"] stack4, _ = arm_maps["MBPP stack-train k=4"] + untr4, _ = arm_maps["MBPP untrained merge k=4 (n=500 rerun)"] + ff1, _ = arm_maps["MBPP trained FF k=1 (n=500 rerun)"] TESTS = [ ("loop k=4 vs k=0, overall", loop0, loop4, None), ("loop k=4 vs k=0, hard", loop0, loop4, mbpp_hard), + ("loop k=4 vs UNTRAINED merge k=4, hard (net effect)", + untr4, loop4, mbpp_hard), + ("loop k=4 vs UNTRAINED merge k=4, overall", untr4, loop4, None), + ("trained FF vs UNTRAINED merge, hard", untr4, ff1, mbpp_hard), + ("distill k=1 vs UNTRAINED merge k=4, hard", untr4, dist1, mbpp_hard), ("distill k=1 vs loop k=4, overall", loop4, dist1, None), ("distill k=1 vs loop k=4, hard", loop4, dist1, mbpp_hard), ("stack-train k=4 vs distill k=1, hard", dist1, stack4, mbpp_hard),