From d81afef4886e78a82fb74d0c77f5278ae4460920 Mon Sep 17 00:00:00 2001 From: Nils Date: Wed, 15 Jul 2026 20:57:44 +0200 Subject: [PATCH] items 16 + 15-closure scored: cross-task transfer toxic (task-locality confirmed 3-point); noise-s0 50%+ was a lucky seed Co-Authored-By: Claude Fable 5 --- results-loop/PROTOCOL_UNIFIED.md | 21 + results-loop/eval_code2gsm_loop.json | 4143 ++++++++++++++++++ results-loop/eval_code2gsm_ns.json | 3109 +++++++++++++ results-loop/eval_code_merge_ns_s1_e400.json | 3036 +++++++++++++ results-loop/eval_code_merge_ns_s2_e400.json | 3036 +++++++++++++ 5 files changed, 13345 insertions(+) create mode 100644 results-loop/eval_code2gsm_loop.json create mode 100644 results-loop/eval_code2gsm_ns.json create mode 100644 results-loop/eval_code_merge_ns_s1_e400.json create mode 100644 results-loop/eval_code_merge_ns_s2_e400.json diff --git a/results-loop/PROTOCOL_UNIFIED.md b/results-loop/PROTOCOL_UNIFIED.md index c98eb09..ca1bebf 100644 --- a/results-loop/PROTOCOL_UNIFIED.md +++ b/results-loop/PROTOCOL_UNIFIED.md @@ -303,3 +303,24 @@ the fidelity refutation replicates. easy 70.5/73.0/72.1 at k=2/4/8 ordinary seed spread at n=28). Two-seed conclusion: contraction-with- free-B loses ~17 points of easy items regardless of seed; the phase diagram's Parcae point is solid. + +--- Outcome, item 16 (code->GSM8K transfer, scored 2026-07-15): ALL THREE +PREDICTIONS CONFIRMED, emphatically. MBPP-trained loop on GSM8K: hard +0.8-1.6% at every k (prediction a: ~0 gain — plan content is task-local); +easy 93.1 -> 27.6-44.8% (prediction b: damaged, in fact WORSE than the +GSM-trained merge's 48%); overall strictly below k=0 at every k>0 +(prediction c). noise-s0 variant identical (easy 34.5, hard 1.6). The +transfer-distance ladder ends cleanly: near (HumanEval) tie, far-code +(LCB) trained-hurts, cross-task (GSM8K) trained-content actively toxic +while gaining nothing. Task-locality of the learned content is now a +three-point monotone result. + +--- Closure of the item-15b/15c "ceiling nudged upward" question +(2026-07-15, after ns seeds): LUCKY SEED, per the pre-registered rule. +noise-s0 hard(k=4) across seeds: 53.6 / 39.3 / 35.7 -> seed mean 42.9, +inside the 37-46 band. Fidelity across seeds intact (easy 90.2-94.3 — +the factorial conclusion is seed-robust); the 50-54% cells (ns seed 0, +h2048 single seed) were upper-tail draws of the same distribution the +merge's 46.4 came from. No recipe amendment; the abstract's original +"same level within noise" framing stands; single-cell records are not +levels — only seed means are. diff --git a/results-loop/eval_code2gsm_loop.json b/results-loop/eval_code2gsm_loop.json new file mode 100644 index 0000000..fefacdf --- /dev/null +++ b/results-loop/eval_code2gsm_loop.json @@ -0,0 +1,4143 @@ +{ + "tag": "code2gsm_loop", + "ks": { + "0": { + "acc": 0.10546875, + "by_label": { + "drop": 0.0, + "easy": 0.9310344827586207, + "hard": 0.0 + }, + "per_item": [ + { + "idx": 1092, + "ok": false + }, + { + "idx": 696, + "ok": false + }, + { + "idx": 242, + "ok": false + }, + { + "idx": 1085, + "ok": true + }, + { + "idx": 510, + "ok": false + }, + { + "idx": 334, + "ok": false + }, + { + "idx": 139, + "ok": false + }, + { + "idx": 849, + "ok": false + }, + { + "idx": 593, + "ok": false + }, + { + "idx": 579, + "ok": false + }, + { + "idx": 1064, + "ok": false + }, + { + "idx": 275, + "ok": false + }, + { + "idx": 1175, + "ok": false + }, + { + "idx": 1071, + "ok": false + }, + { + "idx": 1284, + "ok": false + }, + { + "idx": 430, + "ok": false + }, + { + "idx": 1088, + "ok": false + }, + { + "idx": 215, + "ok": false + }, + { + "idx": 841, + "ok": false + }, + { + "idx": 1299, + "ok": false + }, + { + "idx": 1113, + "ok": true + }, + { + "idx": 826, + "ok": false + }, + { + "idx": 570, + "ok": false + }, + { + "idx": 598, + "ok": false + }, + { + "idx": 905, + "ok": false + }, + { + "idx": 761, + "ok": false + }, + { + "idx": 1163, + "ok": false + }, + { + "idx": 1287, + "ok": false + }, + { + "idx": 282, + "ok": false + }, + { + "idx": 321, + "ok": false + }, + { + "idx": 252, + "ok": false + }, + { + "idx": 246, + "ok": false + }, + { + "idx": 781, + "ok": false + }, + { + "idx": 821, + "ok": false + }, + { + "idx": 1210, + "ok": true + }, + { + "idx": 958, + "ok": true + }, + { + "idx": 285, + "ok": false + }, + { + "idx": 1146, + "ok": false + }, + { + "idx": 612, + "ok": false + }, + { + "idx": 725, + "ok": false + }, + { + "idx": 1294, + "ok": false + }, + { + "idx": 968, + "ok": false + }, + { + "idx": 850, + "ok": false + }, + { + "idx": 446, + "ok": false + }, + { + "idx": 976, + "ok": false + }, + { + "idx": 1001, + "ok": false + }, + { + "idx": 1027, + "ok": true + }, + { + "idx": 651, + "ok": false + }, + { + "idx": 1009, + "ok": false + }, + { + "idx": 123, + "ok": false + }, + { + "idx": 909, + "ok": false + }, + { + "idx": 614, + "ok": false + }, + { + "idx": 292, + "ok": false + }, + { + "idx": 1014, + "ok": false + }, + { + "idx": 107, + "ok": false + }, + { + "idx": 1272, + "ok": false + }, + { + "idx": 441, + "ok": false + }, + { + "idx": 52, + "ok": false + }, + { + "idx": 727, + "ok": false + }, + { + "idx": 965, + "ok": false + }, + { + "idx": 800, + "ok": false + }, + { + "idx": 20, + "ok": false + }, + { + "idx": 1078, + "ok": false + }, + { + "idx": 136, + "ok": false + }, + { + "idx": 166, + "ok": true + }, + { + "idx": 809, + "ok": true + }, + { + "idx": 13, + "ok": false + }, + { + "idx": 739, + "ok": false + }, + { + "idx": 84, + "ok": true + }, + { + "idx": 237, + "ok": true + }, + { + "idx": 1271, + "ok": false + }, + { + "idx": 7, + "ok": false + }, + { + "idx": 553, + "ok": false + }, + { + "idx": 1311, + "ok": false + }, + { + "idx": 599, + "ok": false + }, + { + "idx": 464, + "ok": false + }, + { + "idx": 288, + "ok": false + }, + { + "idx": 1173, + "ok": false + }, + { + "idx": 589, + "ok": false + }, + { + "idx": 391, + "ok": false + }, + { + "idx": 888, + "ok": true + }, + { + "idx": 943, + "ok": false + }, + { + "idx": 676, + "ok": false + }, + { + "idx": 786, + "ok": false + }, + { + "idx": 344, + "ok": true + }, + { + "idx": 677, + "ok": false + }, + { + "idx": 863, + "ok": false + }, + { + "idx": 891, + "ok": false + }, + { + "idx": 303, + "ok": false + }, + { + "idx": 916, + "ok": false + }, + { + "idx": 302, + "ok": false + }, + { + "idx": 1073, + "ok": false + }, + { + "idx": 647, + "ok": false + }, + { + "idx": 264, + "ok": false + }, + { + "idx": 427, + "ok": false + }, + { + "idx": 382, + "ok": false + }, + { + "idx": 715, + "ok": false + }, + { + "idx": 796, + "ok": false + }, + { + "idx": 875, + "ok": false + }, + { + "idx": 1007, + "ok": false + }, + { + "idx": 797, + "ok": true + }, + { + "idx": 450, + "ok": false + }, + { + "idx": 401, + "ok": false + }, + { + "idx": 899, + "ok": false + }, + { + "idx": 418, + "ok": false + }, + { + "idx": 1201, + "ok": false + }, + { + "idx": 101, + "ok": false + }, + { + "idx": 795, + "ok": false + }, + { + "idx": 68, + "ok": false + }, + { + "idx": 479, + "ok": false + }, + { + "idx": 1296, + "ok": false + }, + { + "idx": 173, + "ok": false + }, + { + "idx": 744, + "ok": false + }, + { + "idx": 116, + "ok": true + }, + { + "idx": 1306, + "ok": false + }, + { + "idx": 355, + "ok": false + }, + { + "idx": 477, + "ok": false + }, + { + "idx": 1250, + "ok": false + }, + { + "idx": 609, + "ok": false + }, + { + "idx": 1254, + "ok": true + }, + { + "idx": 177, + "ok": false + }, + { + "idx": 1048, + "ok": false + }, + { + "idx": 582, + "ok": false + }, + { + "idx": 723, + "ok": false + }, + { + "idx": 842, + "ok": false + }, + { + "idx": 938, + "ok": false + }, + { + "idx": 110, + "ok": false + }, + { + "idx": 1057, + "ok": false + }, + { + "idx": 1120, + "ok": false + }, + { + "idx": 881, + "ok": false + }, + { + "idx": 1189, + "ok": false + }, + { + "idx": 931, + "ok": false + }, + { + "idx": 1003, + "ok": false + }, + { + "idx": 521, + "ok": false + }, + { + "idx": 974, + "ok": false + }, + { + "idx": 690, + "ok": false + }, + { + "idx": 544, + "ok": false + }, + { + "idx": 86, + "ok": false + }, + { + "idx": 89, + "ok": false + }, + { + "idx": 333, + "ok": false + }, + { + "idx": 716, + "ok": false + }, + { + "idx": 14, + "ok": false + }, + { + "idx": 287, + "ok": false + }, + { + "idx": 130, + "ok": false + }, + { + "idx": 876, + "ok": false + }, + { + "idx": 454, + "ok": false + }, + { + "idx": 1246, + "ok": true + }, + { + "idx": 812, + "ok": false + }, + { + "idx": 1142, + "ok": false + }, + { + "idx": 452, + "ok": false + }, + { + "idx": 929, + "ok": false + }, + { + "idx": 394, + "ok": false + }, + { + "idx": 695, + "ok": false + }, + { + "idx": 1245, + "ok": false + }, + { + "idx": 209, + "ok": false + }, + { + "idx": 1241, + "ok": false + }, + { + "idx": 175, + "ok": false + }, + { + "idx": 653, + "ok": false + }, + { + "idx": 661, + "ok": false + }, + { + "idx": 1097, + "ok": false + }, + { + "idx": 933, + "ok": false + }, + { + "idx": 665, + "ok": false + }, + { + "idx": 523, + "ok": false + }, + { + "idx": 59, + "ok": false + }, + { + "idx": 1069, + "ok": false + }, + { + "idx": 90, + "ok": false + }, + { + "idx": 389, + "ok": false + }, + { + "idx": 755, + "ok": false + }, + { + "idx": 164, + "ok": true + }, + { + "idx": 429, + "ok": false + }, + { + "idx": 1074, + "ok": false + }, + { + "idx": 708, + "ok": true + }, + { + "idx": 384, + "ok": false + }, + { + "idx": 412, + "ok": false + }, + { + "idx": 514, + "ok": false + }, + { + "idx": 617, + "ok": false + }, + { + "idx": 639, + "ok": false + }, + { + "idx": 1058, + "ok": false + }, + { + "idx": 787, + "ok": false + }, + { + "idx": 987, + "ok": false + }, + { + "idx": 704, + "ok": false + }, + { + "idx": 491, + "ok": false + }, + { + "idx": 91, + "ok": false + }, + { + "idx": 626, + "ok": false + }, + { + "idx": 1130, + "ok": false + }, + { + "idx": 148, + "ok": false + }, + { + "idx": 18, + "ok": false + }, + { + "idx": 897, + "ok": true + }, + { + "idx": 97, + "ok": false + }, + { + "idx": 844, + "ok": false + }, + { + "idx": 1010, + "ok": false + }, + { + "idx": 901, + "ok": false + }, + { + "idx": 241, + "ok": false + }, + { + "idx": 493, + "ok": false + }, + { + "idx": 202, + "ok": true + }, + { + "idx": 314, + "ok": false + }, + { + "idx": 847, + "ok": false + }, + { + "idx": 437, + "ok": false + }, + { + "idx": 902, + "ok": true + }, + { + "idx": 157, + "ok": false + }, + { + "idx": 874, + "ok": true + }, + { + "idx": 1144, + "ok": false + }, + { + "idx": 807, + "ok": false + }, + { + "idx": 80, + "ok": true + }, + { + "idx": 369, + "ok": false + }, + { + "idx": 511, + "ok": false + }, + { + "idx": 262, + "ok": false + }, + { + "idx": 571, + "ok": false + }, + { + "idx": 721, + "ok": false + }, + { + "idx": 654, + "ok": false + }, + { + "idx": 890, + "ok": true + }, + { + "idx": 219, + "ok": false + }, + { + "idx": 1140, + "ok": false + }, + { + "idx": 584, + "ok": false + }, + { + "idx": 1249, + "ok": false + }, + { + "idx": 1108, + "ok": true + }, + { + "idx": 607, + "ok": false + }, + { + "idx": 904, + "ok": false + }, + { + "idx": 1052, + "ok": false + }, + { + "idx": 1238, + "ok": false + }, + { + "idx": 946, + "ok": false + }, + { + "idx": 1098, + "ok": false + }, + { + "idx": 534, + "ok": false + }, + { + "idx": 558, + "ok": false + }, + { + "idx": 474, + "ok": true + }, + { + "idx": 33, + "ok": false + }, + { + "idx": 243, + "ok": false + }, + { + "idx": 1259, + "ok": false + }, + { + "idx": 353, + "ok": false + }, + { + "idx": 509, + "ok": false + }, + { + "idx": 1096, + "ok": false + }, + { + "idx": 1055, + "ok": false + }, + { + "idx": 877, + "ok": false + }, + { + "idx": 249, + "ok": false + }, + { + "idx": 1156, + "ok": false + }, + { + "idx": 735, + "ok": false + }, + { + "idx": 470, + "ok": false + }, + { + "idx": 1119, + "ok": false + }, + { + "idx": 577, + "ok": true + }, + { + "idx": 453, + "ok": false + }, + { + "idx": 132, + "ok": false + }, + { + "idx": 1061, + "ok": false + }, + { + "idx": 629, + "ok": false + }, + { + "idx": 670, + "ok": true + }, + { + "idx": 478, + "ok": false + }, + { + "idx": 764, + "ok": false + }, + { + "idx": 983, + "ok": false + }, + { + "idx": 587, + "ok": false + }, + { + "idx": 1192, + "ok": false + }, + { + "idx": 350, + "ok": false + }, + { + "idx": 281, + "ok": false + }, + { + "idx": 31, + "ok": false + }, + { + "idx": 1132, + "ok": false + }, + { + "idx": 1033, + "ok": false + }, + { + "idx": 671, + "ok": false + }, + { + "idx": 751, + "ok": false + } + ] + }, + "1": { + "acc": 0.0703125, + "by_label": { + "drop": 0.03, + "easy": 0.4482758620689655, + "hard": 0.015748031496062992 + }, + "per_item": [ + { + "idx": 1092, + "ok": false + }, + { + "idx": 696, + "ok": false + }, + { + "idx": 242, + "ok": false + }, + { + "idx": 1085, + "ok": true + }, + { + "idx": 510, + "ok": false + }, + { + "idx": 334, + "ok": false + }, + { + "idx": 139, + "ok": false + }, + { + "idx": 849, + "ok": false + }, + { + "idx": 593, + "ok": false + }, + { + "idx": 579, + "ok": false + }, + { + "idx": 1064, + "ok": false + }, + { + "idx": 275, + "ok": false + }, + { + "idx": 1175, + "ok": false + }, + { + "idx": 1071, + "ok": false + }, + { + "idx": 1284, + "ok": false + }, + { + "idx": 430, + "ok": false + }, + { + "idx": 1088, + "ok": false + }, + { + "idx": 215, + "ok": false + }, + { + "idx": 841, + "ok": false + }, + { + "idx": 1299, + "ok": false + }, + { + "idx": 1113, + "ok": true + }, + { + "idx": 826, + "ok": false + }, + { + "idx": 570, + "ok": false + }, + { + "idx": 598, + "ok": false + }, + { + "idx": 905, + "ok": false + }, + { + "idx": 761, + "ok": false + }, + { + "idx": 1163, + "ok": true + }, + { + "idx": 1287, + "ok": false + }, + { + "idx": 282, + "ok": false + }, + { + "idx": 321, + "ok": false + }, + { + "idx": 252, + "ok": false + }, + { + "idx": 246, + "ok": false + }, + { + "idx": 781, + "ok": false + }, + { + "idx": 821, + "ok": false + }, + { + "idx": 1210, + "ok": true + }, + { + "idx": 958, + "ok": false + }, + { + "idx": 285, + "ok": false + }, + { + "idx": 1146, + "ok": false + }, + { + "idx": 612, + "ok": false + }, + { + "idx": 725, + "ok": false + }, + { + "idx": 1294, + "ok": false + }, + { + "idx": 968, + "ok": false + }, + { + "idx": 850, + "ok": false + }, + { + "idx": 446, + "ok": false + }, + { + "idx": 976, + "ok": false + }, + { + "idx": 1001, + "ok": false + }, + { + "idx": 1027, + "ok": true + }, + { + "idx": 651, + "ok": false + }, + { + "idx": 1009, + "ok": false + }, + { + "idx": 123, + "ok": false + }, + { + "idx": 909, + "ok": false + }, + { + "idx": 614, + "ok": false + }, + { + "idx": 292, + "ok": false + }, + { + "idx": 1014, + "ok": false + }, + { + "idx": 107, + "ok": false + }, + { + "idx": 1272, + "ok": false + }, + { + "idx": 441, + "ok": false + }, + { + "idx": 52, + "ok": false + }, + { + "idx": 727, + "ok": false + }, + { + "idx": 965, + "ok": false + }, + { + "idx": 800, + "ok": false + }, + { + "idx": 20, + "ok": false + }, + { + "idx": 1078, + "ok": false + }, + { + "idx": 136, + "ok": false + }, + { + "idx": 166, + "ok": true + }, + { + "idx": 809, + "ok": true + }, + { + "idx": 13, + "ok": false + }, + { + "idx": 739, + "ok": false + }, + { + "idx": 84, + "ok": true + }, + { + "idx": 237, + "ok": false + }, + { + "idx": 1271, + "ok": false + }, + { + "idx": 7, + "ok": false + }, + { + "idx": 553, + "ok": false + }, + { + "idx": 1311, + "ok": false + }, + { + "idx": 599, + "ok": false + }, + { + "idx": 464, + "ok": false + }, + { + "idx": 288, + "ok": false + }, + { + "idx": 1173, + "ok": false + }, + { + "idx": 589, + "ok": false + }, + { + "idx": 391, + "ok": false + }, + { + "idx": 888, + "ok": true + }, + { + "idx": 943, + "ok": false + }, + { + "idx": 676, + "ok": false + }, + { + "idx": 786, + "ok": false + }, + { + "idx": 344, + "ok": false + }, + { + "idx": 677, + "ok": false + }, + { + "idx": 863, + "ok": false + }, + { + "idx": 891, + "ok": false + }, + { + "idx": 303, + "ok": false + }, + { + "idx": 916, + "ok": false + }, + { + "idx": 302, + "ok": false + }, + { + "idx": 1073, + "ok": true + }, + { + "idx": 647, + "ok": false + }, + { + "idx": 264, + "ok": false + }, + { + "idx": 427, + "ok": false + }, + { + "idx": 382, + "ok": false + }, + { + "idx": 715, + "ok": false + }, + { + "idx": 796, + "ok": false + }, + { + "idx": 875, + "ok": false + }, + { + "idx": 1007, + "ok": false + }, + { + "idx": 797, + "ok": false + }, + { + "idx": 450, + "ok": false + }, + { + "idx": 401, + "ok": false + }, + { + "idx": 899, + "ok": false + }, + { + "idx": 418, + "ok": false + }, + { + "idx": 1201, + "ok": false + }, + { + "idx": 101, + "ok": false + }, + { + "idx": 795, + "ok": false + }, + { + "idx": 68, + "ok": false + }, + { + "idx": 479, + "ok": false + }, + { + "idx": 1296, + "ok": false + }, + { + "idx": 173, + "ok": false + }, + { + "idx": 744, + "ok": false + }, + { + "idx": 116, + "ok": false + }, + { + "idx": 1306, + "ok": true + }, + { + "idx": 355, + "ok": false + }, + { + "idx": 477, + "ok": false + }, + { + "idx": 1250, + "ok": false + }, + { + "idx": 609, + "ok": false + }, + { + "idx": 1254, + "ok": false + }, + { + "idx": 177, + "ok": false + }, + { + "idx": 1048, + "ok": false + }, + { + "idx": 582, + "ok": false + }, + { + "idx": 723, + "ok": false + }, + { + "idx": 842, + "ok": false + }, + { + "idx": 938, + "ok": false + }, + { + "idx": 110, + "ok": false + }, + { + "idx": 1057, + "ok": false + }, + { + "idx": 1120, + "ok": true + }, + { + "idx": 881, + "ok": false + }, + { + "idx": 1189, + "ok": false + }, + { + "idx": 931, + "ok": false + }, + { + "idx": 1003, + "ok": false + }, + { + "idx": 521, + "ok": false + }, + { + "idx": 974, + "ok": false + }, + { + "idx": 690, + "ok": false + }, + { + "idx": 544, + "ok": false + }, + { + "idx": 86, + "ok": false + }, + { + "idx": 89, + "ok": true + }, + { + "idx": 333, + "ok": false + }, + { + "idx": 716, + "ok": false + }, + { + "idx": 14, + "ok": false + }, + { + "idx": 287, + "ok": false + }, + { + "idx": 130, + "ok": false + }, + { + "idx": 876, + "ok": false + }, + { + "idx": 454, + "ok": false + }, + { + "idx": 1246, + "ok": false + }, + { + "idx": 812, + "ok": false + }, + { + "idx": 1142, + "ok": false + }, + { + "idx": 452, + "ok": false + }, + { + "idx": 929, + "ok": false + }, + { + "idx": 394, + "ok": false + }, + { + "idx": 695, + "ok": false + }, + { + "idx": 1245, + "ok": false + }, + { + "idx": 209, + "ok": false + }, + { + "idx": 1241, + "ok": false + }, + { + "idx": 175, + "ok": false + }, + { + "idx": 653, + "ok": false + }, + { + "idx": 661, + "ok": false + }, + { + "idx": 1097, + "ok": false + }, + { + "idx": 933, + "ok": false + }, + { + "idx": 665, + "ok": false + }, + { + "idx": 523, + "ok": false + }, + { + "idx": 59, + "ok": false + }, + { + "idx": 1069, + "ok": false + }, + { + "idx": 90, + "ok": false + }, + { + "idx": 389, + "ok": false + }, + { + "idx": 755, + "ok": false + }, + { + "idx": 164, + "ok": false + }, + { + "idx": 429, + "ok": false + }, + { + "idx": 1074, + "ok": false + }, + { + "idx": 708, + "ok": true + }, + { + "idx": 384, + "ok": false + }, + { + "idx": 412, + "ok": false + }, + { + "idx": 514, + "ok": false + }, + { + "idx": 617, + "ok": false + }, + { + "idx": 639, + "ok": false + }, + { + "idx": 1058, + "ok": false + }, + { + "idx": 787, + "ok": false + }, + { + "idx": 987, + "ok": false + }, + { + "idx": 704, + "ok": false + }, + { + "idx": 491, + "ok": false + }, + { + "idx": 91, + "ok": false + }, + { + "idx": 626, + "ok": false + }, + { + "idx": 1130, + "ok": false + }, + { + "idx": 148, + "ok": false + }, + { + "idx": 18, + "ok": false + }, + { + "idx": 897, + "ok": false + }, + { + "idx": 97, + "ok": false + }, + { + "idx": 844, + "ok": false + }, + { + "idx": 1010, + "ok": false + }, + { + "idx": 901, + "ok": false + }, + { + "idx": 241, + "ok": false + }, + { + "idx": 493, + "ok": false + }, + { + "idx": 202, + "ok": true + }, + { + "idx": 314, + "ok": false + }, + { + "idx": 847, + "ok": false + }, + { + "idx": 437, + "ok": false + }, + { + "idx": 902, + "ok": true + }, + { + "idx": 157, + "ok": false + }, + { + "idx": 874, + "ok": false + }, + { + "idx": 1144, + "ok": false + }, + { + "idx": 807, + "ok": false + }, + { + "idx": 80, + "ok": true + }, + { + "idx": 369, + "ok": false + }, + { + "idx": 511, + "ok": false + }, + { + "idx": 262, + "ok": false + }, + { + "idx": 571, + "ok": false + }, + { + "idx": 721, + "ok": false + }, + { + "idx": 654, + "ok": false + }, + { + "idx": 890, + "ok": false + }, + { + "idx": 219, + "ok": false + }, + { + "idx": 1140, + "ok": false + }, + { + "idx": 584, + "ok": false + }, + { + "idx": 1249, + "ok": false + }, + { + "idx": 1108, + "ok": true + }, + { + "idx": 607, + "ok": false + }, + { + "idx": 904, + "ok": false + }, + { + "idx": 1052, + "ok": false + }, + { + "idx": 1238, + "ok": false + }, + { + "idx": 946, + "ok": false + }, + { + "idx": 1098, + "ok": false + }, + { + "idx": 534, + "ok": false + }, + { + "idx": 558, + "ok": false + }, + { + "idx": 474, + "ok": false + }, + { + "idx": 33, + "ok": false + }, + { + "idx": 243, + "ok": false + }, + { + "idx": 1259, + "ok": false + }, + { + "idx": 353, + "ok": false + }, + { + "idx": 509, + "ok": false + }, + { + "idx": 1096, + "ok": false + }, + { + "idx": 1055, + "ok": false + }, + { + "idx": 877, + "ok": false + }, + { + "idx": 249, + "ok": false + }, + { + "idx": 1156, + "ok": false + }, + { + "idx": 735, + "ok": false + }, + { + "idx": 470, + "ok": false + }, + { + "idx": 1119, + "ok": false + }, + { + "idx": 577, + "ok": false + }, + { + "idx": 453, + "ok": false + }, + { + "idx": 132, + "ok": false + }, + { + "idx": 1061, + "ok": false + }, + { + "idx": 629, + "ok": false + }, + { + "idx": 670, + "ok": false + }, + { + "idx": 478, + "ok": false + }, + { + "idx": 764, + "ok": false + }, + { + "idx": 983, + "ok": false + }, + { + "idx": 587, + "ok": false + }, + { + "idx": 1192, + "ok": false + }, + { + "idx": 350, + "ok": false + }, + { + "idx": 281, + "ok": false + }, + { + "idx": 31, + "ok": false + }, + { + "idx": 1132, + "ok": false + }, + { + "idx": 1033, + "ok": false + }, + { + "idx": 671, + "ok": false + }, + { + "idx": 751, + "ok": false + } + ] + }, + "2": { + "acc": 0.046875, + "by_label": { + "drop": 0.02, + "easy": 0.27586206896551724, + "hard": 0.015748031496062992 + }, + "per_item": [ + { + "idx": 1092, + "ok": false + }, + { + "idx": 696, + "ok": false + }, + { + "idx": 242, + "ok": false + }, + { + "idx": 1085, + "ok": true + }, + { + "idx": 510, + "ok": false + }, + { + "idx": 334, + "ok": false + }, + { + "idx": 139, + "ok": false + }, + { + "idx": 849, + "ok": false + }, + { + "idx": 593, + "ok": false + }, + { + "idx": 579, + "ok": false + }, + { + "idx": 1064, + "ok": false + }, + { + "idx": 275, + "ok": false + }, + { + "idx": 1175, + "ok": false + }, + { + "idx": 1071, + "ok": false + }, + { + "idx": 1284, + "ok": false + }, + { + "idx": 430, + "ok": false + }, + { + "idx": 1088, + "ok": false + }, + { + "idx": 215, + "ok": false + }, + { + "idx": 841, + "ok": false + }, + { + "idx": 1299, + "ok": false + }, + { + "idx": 1113, + "ok": true + }, + { + "idx": 826, + "ok": false + }, + { + "idx": 570, + "ok": false + }, + { + "idx": 598, + "ok": false + }, + { + "idx": 905, + "ok": false + }, + { + "idx": 761, + "ok": false + }, + { + "idx": 1163, + "ok": true + }, + { + "idx": 1287, + "ok": false + }, + { + "idx": 282, + "ok": false + }, + { + "idx": 321, + "ok": false + }, + { + "idx": 252, + "ok": false + }, + { + "idx": 246, + "ok": false + }, + { + "idx": 781, + "ok": false + }, + { + "idx": 821, + "ok": false + }, + { + "idx": 1210, + "ok": false + }, + { + "idx": 958, + "ok": false + }, + { + "idx": 285, + "ok": false + }, + { + "idx": 1146, + "ok": false + }, + { + "idx": 612, + "ok": false + }, + { + "idx": 725, + "ok": false + }, + { + "idx": 1294, + "ok": false + }, + { + "idx": 968, + "ok": false + }, + { + "idx": 850, + "ok": false + }, + { + "idx": 446, + "ok": false + }, + { + "idx": 976, + "ok": false + }, + { + "idx": 1001, + "ok": false + }, + { + "idx": 1027, + "ok": false + }, + { + "idx": 651, + "ok": false + }, + { + "idx": 1009, + "ok": false + }, + { + "idx": 123, + "ok": false + }, + { + "idx": 909, + "ok": false + }, + { + "idx": 614, + "ok": false + }, + { + "idx": 292, + "ok": false + }, + { + "idx": 1014, + "ok": false + }, + { + "idx": 107, + "ok": false + }, + { + "idx": 1272, + "ok": false + }, + { + "idx": 441, + "ok": false + }, + { + "idx": 52, + "ok": false + }, + { + "idx": 727, + "ok": false + }, + { + "idx": 965, + "ok": false + }, + { + "idx": 800, + "ok": false + }, + { + "idx": 20, + "ok": false + }, + { + "idx": 1078, + "ok": false + }, + { + "idx": 136, + "ok": false + }, + { + "idx": 166, + "ok": true + }, + { + "idx": 809, + "ok": false + }, + { + "idx": 13, + "ok": false + }, + { + "idx": 739, + "ok": false + }, + { + "idx": 84, + "ok": true + }, + { + "idx": 237, + "ok": false + }, + { + "idx": 1271, + "ok": false + }, + { + "idx": 7, + "ok": false + }, + { + "idx": 553, + "ok": false + }, + { + "idx": 1311, + "ok": false + }, + { + "idx": 599, + "ok": false + }, + { + "idx": 464, + "ok": false + }, + { + "idx": 288, + "ok": true + }, + { + "idx": 1173, + "ok": false + }, + { + "idx": 589, + "ok": false + }, + { + "idx": 391, + "ok": false + }, + { + "idx": 888, + "ok": true + }, + { + "idx": 943, + "ok": false + }, + { + "idx": 676, + "ok": false + }, + { + "idx": 786, + "ok": false + }, + { + "idx": 344, + "ok": false + }, + { + "idx": 677, + "ok": false + }, + { + "idx": 863, + "ok": false + }, + { + "idx": 891, + "ok": false + }, + { + "idx": 303, + "ok": false + }, + { + "idx": 916, + "ok": false + }, + { + "idx": 302, + "ok": false + }, + { + "idx": 1073, + "ok": false + }, + { + "idx": 647, + "ok": false + }, + { + "idx": 264, + "ok": false + }, + { + "idx": 427, + "ok": false + }, + { + "idx": 382, + "ok": false + }, + { + "idx": 715, + "ok": false + }, + { + "idx": 796, + "ok": false + }, + { + "idx": 875, + "ok": false + }, + { + "idx": 1007, + "ok": false + }, + { + "idx": 797, + "ok": false + }, + { + "idx": 450, + "ok": false + }, + { + "idx": 401, + "ok": false + }, + { + "idx": 899, + "ok": false + }, + { + "idx": 418, + "ok": false + }, + { + "idx": 1201, + "ok": false + }, + { + "idx": 101, + "ok": false + }, + { + "idx": 795, + "ok": false + }, + { + "idx": 68, + "ok": false + }, + { + "idx": 479, + "ok": false + }, + { + "idx": 1296, + "ok": false + }, + { + "idx": 173, + "ok": false + }, + { + "idx": 744, + "ok": false + }, + { + "idx": 116, + "ok": false + }, + { + "idx": 1306, + "ok": true + }, + { + "idx": 355, + "ok": false + }, + { + "idx": 477, + "ok": false + }, + { + "idx": 1250, + "ok": false + }, + { + "idx": 609, + "ok": false + }, + { + "idx": 1254, + "ok": false + }, + { + "idx": 177, + "ok": false + }, + { + "idx": 1048, + "ok": false + }, + { + "idx": 582, + "ok": false + }, + { + "idx": 723, + "ok": false + }, + { + "idx": 842, + "ok": false + }, + { + "idx": 938, + "ok": false + }, + { + "idx": 110, + "ok": false + }, + { + "idx": 1057, + "ok": false + }, + { + "idx": 1120, + "ok": false + }, + { + "idx": 881, + "ok": false + }, + { + "idx": 1189, + "ok": false + }, + { + "idx": 931, + "ok": false + }, + { + "idx": 1003, + "ok": false + }, + { + "idx": 521, + "ok": false + }, + { + "idx": 974, + "ok": false + }, + { + "idx": 690, + "ok": false + }, + { + "idx": 544, + "ok": false + }, + { + "idx": 86, + "ok": false + }, + { + "idx": 89, + "ok": true + }, + { + "idx": 333, + "ok": false + }, + { + "idx": 716, + "ok": false + }, + { + "idx": 14, + "ok": false + }, + { + "idx": 287, + "ok": false + }, + { + "idx": 130, + "ok": false + }, + { + "idx": 876, + "ok": false + }, + { + "idx": 454, + "ok": false + }, + { + "idx": 1246, + "ok": false + }, + { + "idx": 812, + "ok": false + }, + { + "idx": 1142, + "ok": false + }, + { + "idx": 452, + "ok": false + }, + { + "idx": 929, + "ok": false + }, + { + "idx": 394, + "ok": false + }, + { + "idx": 695, + "ok": false + }, + { + "idx": 1245, + "ok": false + }, + { + "idx": 209, + "ok": false + }, + { + "idx": 1241, + "ok": false + }, + { + "idx": 175, + "ok": false + }, + { + "idx": 653, + "ok": false + }, + { + "idx": 661, + "ok": false + }, + { + "idx": 1097, + "ok": false + }, + { + "idx": 933, + "ok": false + }, + { + "idx": 665, + "ok": false + }, + { + "idx": 523, + "ok": false + }, + { + "idx": 59, + "ok": false + }, + { + "idx": 1069, + "ok": false + }, + { + "idx": 90, + "ok": false + }, + { + "idx": 389, + "ok": false + }, + { + "idx": 755, + "ok": false + }, + { + "idx": 164, + "ok": false + }, + { + "idx": 429, + "ok": false + }, + { + "idx": 1074, + "ok": false + }, + { + "idx": 708, + "ok": true + }, + { + "idx": 384, + "ok": false + }, + { + "idx": 412, + "ok": false + }, + { + "idx": 514, + "ok": false + }, + { + "idx": 617, + "ok": false + }, + { + "idx": 639, + "ok": false + }, + { + "idx": 1058, + "ok": false + }, + { + "idx": 787, + "ok": false + }, + { + "idx": 987, + "ok": false + }, + { + "idx": 704, + "ok": false + }, + { + "idx": 491, + "ok": false + }, + { + "idx": 91, + "ok": false + }, + { + "idx": 626, + "ok": false + }, + { + "idx": 1130, + "ok": false + }, + { + "idx": 148, + "ok": false + }, + { + "idx": 18, + "ok": false + }, + { + "idx": 897, + "ok": false + }, + { + "idx": 97, + "ok": false + }, + { + "idx": 844, + "ok": false + }, + { + "idx": 1010, + "ok": false + }, + { + "idx": 901, + "ok": false + }, + { + "idx": 241, + "ok": false + }, + { + "idx": 493, + "ok": false + }, + { + "idx": 202, + "ok": false + }, + { + "idx": 314, + "ok": false + }, + { + "idx": 847, + "ok": false + }, + { + "idx": 437, + "ok": false + }, + { + "idx": 902, + "ok": true + }, + { + "idx": 157, + "ok": false + }, + { + "idx": 874, + "ok": true + }, + { + "idx": 1144, + "ok": false + }, + { + "idx": 807, + "ok": false + }, + { + "idx": 80, + "ok": false + }, + { + "idx": 369, + "ok": false + }, + { + "idx": 511, + "ok": false + }, + { + "idx": 262, + "ok": false + }, + { + "idx": 571, + "ok": false + }, + { + "idx": 721, + "ok": false + }, + { + "idx": 654, + "ok": false + }, + { + "idx": 890, + "ok": false + }, + { + "idx": 219, + "ok": false + }, + { + "idx": 1140, + "ok": false + }, + { + "idx": 584, + "ok": false + }, + { + "idx": 1249, + "ok": false + }, + { + "idx": 1108, + "ok": false + }, + { + "idx": 607, + "ok": false + }, + { + "idx": 904, + "ok": false + }, + { + "idx": 1052, + "ok": false + }, + { + "idx": 1238, + "ok": false + }, + { + "idx": 946, + "ok": false + }, + { + "idx": 1098, + "ok": false + }, + { + "idx": 534, + "ok": false + }, + { + "idx": 558, + "ok": false + }, + { + "idx": 474, + "ok": false + }, + { + "idx": 33, + "ok": false + }, + { + "idx": 243, + "ok": false + }, + { + "idx": 1259, + "ok": false + }, + { + "idx": 353, + "ok": false + }, + { + "idx": 509, + "ok": false + }, + { + "idx": 1096, + "ok": false + }, + { + "idx": 1055, + "ok": false + }, + { + "idx": 877, + "ok": false + }, + { + "idx": 249, + "ok": false + }, + { + "idx": 1156, + "ok": false + }, + { + "idx": 735, + "ok": false + }, + { + "idx": 470, + "ok": false + }, + { + "idx": 1119, + "ok": false + }, + { + "idx": 577, + "ok": false + }, + { + "idx": 453, + "ok": false + }, + { + "idx": 132, + "ok": false + }, + { + "idx": 1061, + "ok": false + }, + { + "idx": 629, + "ok": false + }, + { + "idx": 670, + "ok": false + }, + { + "idx": 478, + "ok": false + }, + { + "idx": 764, + "ok": false + }, + { + "idx": 983, + "ok": false + }, + { + "idx": 587, + "ok": false + }, + { + "idx": 1192, + "ok": false + }, + { + "idx": 350, + "ok": false + }, + { + "idx": 281, + "ok": false + }, + { + "idx": 31, + "ok": false + }, + { + "idx": 1132, + "ok": false + }, + { + "idx": 1033, + "ok": false + }, + { + "idx": 671, + "ok": false + }, + { + "idx": 751, + "ok": false + } + ] + }, + "4": { + "acc": 0.04296875, + "by_label": { + "drop": 0.02, + "easy": 0.27586206896551724, + "hard": 0.007874015748031496 + }, + "per_item": [ + { + "idx": 1092, + "ok": false + }, + { + "idx": 696, + "ok": false + }, + { + "idx": 242, + "ok": false + }, + { + "idx": 1085, + "ok": true + }, + { + "idx": 510, + "ok": false + }, + { + "idx": 334, + "ok": false + }, + { + "idx": 139, + "ok": false + }, + { + "idx": 849, + "ok": false + }, + { + "idx": 593, + "ok": false + }, + { + "idx": 579, + "ok": false + }, + { + "idx": 1064, + "ok": false + }, + { + "idx": 275, + "ok": false + }, + { + "idx": 1175, + "ok": false + }, + { + "idx": 1071, + "ok": false + }, + { + "idx": 1284, + "ok": false + }, + { + "idx": 430, + "ok": false + }, + { + "idx": 1088, + "ok": false + }, + { + "idx": 215, + "ok": false + }, + { + "idx": 841, + "ok": false + }, + { + "idx": 1299, + "ok": false + }, + { + "idx": 1113, + "ok": true + }, + { + "idx": 826, + "ok": false + }, + { + "idx": 570, + "ok": false + }, + { + "idx": 598, + "ok": false + }, + { + "idx": 905, + "ok": false + }, + { + "idx": 761, + "ok": false + }, + { + "idx": 1163, + "ok": false + }, + { + "idx": 1287, + "ok": false + }, + { + "idx": 282, + "ok": false + }, + { + "idx": 321, + "ok": false + }, + { + "idx": 252, + "ok": false + }, + { + "idx": 246, + "ok": false + }, + { + "idx": 781, + "ok": false + }, + { + "idx": 821, + "ok": false + }, + { + "idx": 1210, + "ok": false + }, + { + "idx": 958, + "ok": false + }, + { + "idx": 285, + "ok": false + }, + { + "idx": 1146, + "ok": false + }, + { + "idx": 612, + "ok": false + }, + { + "idx": 725, + "ok": false + }, + { + "idx": 1294, + "ok": false + }, + { + "idx": 968, + "ok": false + }, + { + "idx": 850, + "ok": false + }, + { + "idx": 446, + "ok": false + }, + { + "idx": 976, + "ok": false + }, + { + "idx": 1001, + "ok": false + }, + { + "idx": 1027, + "ok": true + }, + { + "idx": 651, + "ok": false + }, + { + "idx": 1009, + "ok": false + }, + { + "idx": 123, + "ok": false + }, + { + "idx": 909, + "ok": false + }, + { + "idx": 614, + "ok": false + }, + { + "idx": 292, + "ok": false + }, + { + "idx": 1014, + "ok": false + }, + { + "idx": 107, + "ok": false + }, + { + "idx": 1272, + "ok": false + }, + { + "idx": 441, + "ok": false + }, + { + "idx": 52, + "ok": false + }, + { + "idx": 727, + "ok": false + }, + { + "idx": 965, + "ok": false + }, + { + "idx": 800, + "ok": false + }, + { + "idx": 20, + "ok": false + }, + { + "idx": 1078, + "ok": false + }, + { + "idx": 136, + "ok": false + }, + { + "idx": 166, + "ok": true + }, + { + "idx": 809, + "ok": false + }, + { + "idx": 13, + "ok": false + }, + { + "idx": 739, + "ok": false + }, + { + "idx": 84, + "ok": false + }, + { + "idx": 237, + "ok": false + }, + { + "idx": 1271, + "ok": false + }, + { + "idx": 7, + "ok": false + }, + { + "idx": 553, + "ok": false + }, + { + "idx": 1311, + "ok": false + }, + { + "idx": 599, + "ok": false + }, + { + "idx": 464, + "ok": false + }, + { + "idx": 288, + "ok": false + }, + { + "idx": 1173, + "ok": false + }, + { + "idx": 589, + "ok": false + }, + { + "idx": 391, + "ok": false + }, + { + "idx": 888, + "ok": true + }, + { + "idx": 943, + "ok": false + }, + { + "idx": 676, + "ok": false + }, + { + "idx": 786, + "ok": false + }, + { + "idx": 344, + "ok": false + }, + { + "idx": 677, + "ok": false + }, + { + "idx": 863, + "ok": false + }, + { + "idx": 891, + "ok": false + }, + { + "idx": 303, + "ok": false + }, + { + "idx": 916, + "ok": false + }, + { + "idx": 302, + "ok": false + }, + { + "idx": 1073, + "ok": false + }, + { + "idx": 647, + "ok": false + }, + { + "idx": 264, + "ok": false + }, + { + "idx": 427, + "ok": false + }, + { + "idx": 382, + "ok": false + }, + { + "idx": 715, + "ok": false + }, + { + "idx": 796, + "ok": false + }, + { + "idx": 875, + "ok": false + }, + { + "idx": 1007, + "ok": false + }, + { + "idx": 797, + "ok": false + }, + { + "idx": 450, + "ok": false + }, + { + "idx": 401, + "ok": false + }, + { + "idx": 899, + "ok": false + }, + { + "idx": 418, + "ok": true + }, + { + "idx": 1201, + "ok": false + }, + { + "idx": 101, + "ok": false + }, + { + "idx": 795, + "ok": false + }, + { + "idx": 68, + "ok": false + }, + { + "idx": 479, + "ok": false + }, + { + "idx": 1296, + "ok": false + }, + { + "idx": 173, + "ok": false + }, + { + "idx": 744, + "ok": false + }, + { + "idx": 116, + "ok": false + }, + { + "idx": 1306, + "ok": true + }, + { + "idx": 355, + "ok": false + }, + { + "idx": 477, + "ok": false + }, + { + "idx": 1250, + "ok": false + }, + { + "idx": 609, + "ok": false + }, + { + "idx": 1254, + "ok": false + }, + { + "idx": 177, + "ok": false + }, + { + "idx": 1048, + "ok": false + }, + { + "idx": 582, + "ok": false + }, + { + "idx": 723, + "ok": false + }, + { + "idx": 842, + "ok": false + }, + { + "idx": 938, + "ok": false + }, + { + "idx": 110, + "ok": false + }, + { + "idx": 1057, + "ok": false + }, + { + "idx": 1120, + "ok": false + }, + { + "idx": 881, + "ok": false + }, + { + "idx": 1189, + "ok": false + }, + { + "idx": 931, + "ok": false + }, + { + "idx": 1003, + "ok": false + }, + { + "idx": 521, + "ok": false + }, + { + "idx": 974, + "ok": false + }, + { + "idx": 690, + "ok": false + }, + { + "idx": 544, + "ok": false + }, + { + "idx": 86, + "ok": false + }, + { + "idx": 89, + "ok": true + }, + { + "idx": 333, + "ok": false + }, + { + "idx": 716, + "ok": false + }, + { + "idx": 14, + "ok": false + }, + { + "idx": 287, + "ok": false + }, + { + "idx": 130, + "ok": false + }, + { + "idx": 876, + "ok": false + }, + { + "idx": 454, + "ok": false + }, + { + "idx": 1246, + "ok": false + }, + { + "idx": 812, + "ok": false + }, + { + "idx": 1142, + "ok": false + }, + { + "idx": 452, + "ok": false + }, + { + "idx": 929, + "ok": false + }, + { + "idx": 394, + "ok": false + }, + { + "idx": 695, + "ok": false + }, + { + "idx": 1245, + "ok": false + }, + { + "idx": 209, + "ok": false + }, + { + "idx": 1241, + "ok": false + }, + { + "idx": 175, + "ok": false + }, + { + "idx": 653, + "ok": false + }, + { + "idx": 661, + "ok": false + }, + { + "idx": 1097, + "ok": false + }, + { + "idx": 933, + "ok": false + }, + { + "idx": 665, + "ok": false + }, + { + "idx": 523, + "ok": false + }, + { + "idx": 59, + "ok": false + }, + { + "idx": 1069, + "ok": false + }, + { + "idx": 90, + "ok": false + }, + { + "idx": 389, + "ok": false + }, + { + "idx": 755, + "ok": false + }, + { + "idx": 164, + "ok": false + }, + { + "idx": 429, + "ok": false + }, + { + "idx": 1074, + "ok": false + }, + { + "idx": 708, + "ok": true + }, + { + "idx": 384, + "ok": false + }, + { + "idx": 412, + "ok": false + }, + { + "idx": 514, + "ok": false + }, + { + "idx": 617, + "ok": false + }, + { + "idx": 639, + "ok": false + }, + { + "idx": 1058, + "ok": false + }, + { + "idx": 787, + "ok": false + }, + { + "idx": 987, + "ok": false + }, + { + "idx": 704, + "ok": false + }, + { + "idx": 491, + "ok": false + }, + { + "idx": 91, + "ok": false + }, + { + "idx": 626, + "ok": false + }, + { + "idx": 1130, + "ok": false + }, + { + "idx": 148, + "ok": false + }, + { + "idx": 18, + "ok": false + }, + { + "idx": 897, + "ok": false + }, + { + "idx": 97, + "ok": false + }, + { + "idx": 844, + "ok": false + }, + { + "idx": 1010, + "ok": false + }, + { + "idx": 901, + "ok": false + }, + { + "idx": 241, + "ok": false + }, + { + "idx": 493, + "ok": false + }, + { + "idx": 202, + "ok": false + }, + { + "idx": 314, + "ok": false + }, + { + "idx": 847, + "ok": false + }, + { + "idx": 437, + "ok": false + }, + { + "idx": 902, + "ok": true + }, + { + "idx": 157, + "ok": false + }, + { + "idx": 874, + "ok": false + }, + { + "idx": 1144, + "ok": false + }, + { + "idx": 807, + "ok": false + }, + { + "idx": 80, + "ok": false + }, + { + "idx": 369, + "ok": false + }, + { + "idx": 511, + "ok": false + }, + { + "idx": 262, + "ok": false + }, + { + "idx": 571, + "ok": false + }, + { + "idx": 721, + "ok": false + }, + { + "idx": 654, + "ok": false + }, + { + "idx": 890, + "ok": false + }, + { + "idx": 219, + "ok": false + }, + { + "idx": 1140, + "ok": false + }, + { + "idx": 584, + "ok": false + }, + { + "idx": 1249, + "ok": false + }, + { + "idx": 1108, + "ok": true + }, + { + "idx": 607, + "ok": false + }, + { + "idx": 904, + "ok": false + }, + { + "idx": 1052, + "ok": false + }, + { + "idx": 1238, + "ok": false + }, + { + "idx": 946, + "ok": false + }, + { + "idx": 1098, + "ok": false + }, + { + "idx": 534, + "ok": false + }, + { + "idx": 558, + "ok": false + }, + { + "idx": 474, + "ok": false + }, + { + "idx": 33, + "ok": false + }, + { + "idx": 243, + "ok": false + }, + { + "idx": 1259, + "ok": false + }, + { + "idx": 353, + "ok": false + }, + { + "idx": 509, + "ok": false + }, + { + "idx": 1096, + "ok": false + }, + { + "idx": 1055, + "ok": false + }, + { + "idx": 877, + "ok": false + }, + { + "idx": 249, + "ok": false + }, + { + "idx": 1156, + "ok": false + }, + { + "idx": 735, + "ok": false + }, + { + "idx": 470, + "ok": false + }, + { + "idx": 1119, + "ok": false + }, + { + "idx": 577, + "ok": false + }, + { + "idx": 453, + "ok": false + }, + { + "idx": 132, + "ok": false + }, + { + "idx": 1061, + "ok": false + }, + { + "idx": 629, + "ok": false + }, + { + "idx": 670, + "ok": false + }, + { + "idx": 478, + "ok": false + }, + { + "idx": 764, + "ok": false + }, + { + "idx": 983, + "ok": false + }, + { + "idx": 587, + "ok": false + }, + { + "idx": 1192, + "ok": false + }, + { + "idx": 350, + "ok": false + }, + { + "idx": 281, + "ok": false + }, + { + "idx": 31, + "ok": false + }, + { + "idx": 1132, + "ok": false + }, + { + "idx": 1033, + "ok": false + }, + { + "idx": 671, + "ok": false + }, + { + "idx": 751, + "ok": false + } + ] + } + }, + "n": 256, + "spider": [] +} \ No newline at end of file diff --git a/results-loop/eval_code2gsm_ns.json b/results-loop/eval_code2gsm_ns.json new file mode 100644 index 0000000..80f6ae9 --- /dev/null +++ b/results-loop/eval_code2gsm_ns.json @@ -0,0 +1,3109 @@ +{ + "tag": "code2gsm_ns", + "ks": { + "0": { + "acc": 0.10546875, + "by_label": { + "drop": 0.0, + "easy": 0.9310344827586207, + "hard": 0.0 + }, + "per_item": [ + { + "idx": 1092, + "ok": false + }, + { + "idx": 696, + "ok": false + }, + { + "idx": 242, + "ok": false + }, + { + "idx": 1085, + "ok": true + }, + { + "idx": 510, + "ok": false + }, + { + "idx": 334, + "ok": false + }, + { + "idx": 139, + "ok": false + }, + { + "idx": 849, + "ok": false + }, + { + "idx": 593, + "ok": false + }, + { + "idx": 579, + "ok": false + }, + { + "idx": 1064, + "ok": false + }, + { + "idx": 275, + "ok": false + }, + { + "idx": 1175, + "ok": false + }, + { + "idx": 1071, + "ok": false + }, + { + "idx": 1284, + "ok": false + }, + { + "idx": 430, + "ok": false + }, + { + "idx": 1088, + "ok": false + }, + { + "idx": 215, + "ok": false + }, + { + "idx": 841, + "ok": false + }, + { + "idx": 1299, + "ok": false + }, + { + "idx": 1113, + "ok": true + }, + { + "idx": 826, + "ok": false + }, + { + "idx": 570, + "ok": false + }, + { + "idx": 598, + "ok": false + }, + { + "idx": 905, + "ok": false + }, + { + "idx": 761, + "ok": false + }, + { + "idx": 1163, + "ok": false + }, + { + "idx": 1287, + "ok": false + }, + { + "idx": 282, + "ok": false + }, + { + "idx": 321, + "ok": false + }, + { + "idx": 252, + "ok": false + }, + { + "idx": 246, + "ok": false + }, + { + "idx": 781, + "ok": false + }, + { + "idx": 821, + "ok": false + }, + { + "idx": 1210, + "ok": true + }, + { + "idx": 958, + "ok": true + }, + { + "idx": 285, + "ok": false + }, + { + "idx": 1146, + "ok": false + }, + { + "idx": 612, + "ok": false + }, + { + "idx": 725, + "ok": false + }, + { + "idx": 1294, + "ok": false + }, + { + "idx": 968, + "ok": false + }, + { + "idx": 850, + "ok": false + }, + { + "idx": 446, + "ok": false + }, + { + "idx": 976, + "ok": false + }, + { + "idx": 1001, + "ok": false + }, + { + "idx": 1027, + "ok": true + }, + { + "idx": 651, + "ok": false + }, + { + "idx": 1009, + "ok": false + }, + { + "idx": 123, + "ok": false + }, + { + "idx": 909, + "ok": false + }, + { + "idx": 614, + "ok": false + }, + { + "idx": 292, + "ok": false + }, + { + "idx": 1014, + "ok": false + }, + { + "idx": 107, + "ok": false + }, + { + "idx": 1272, + "ok": false + }, + { + "idx": 441, + "ok": false + }, + { + "idx": 52, + "ok": false + }, + { + "idx": 727, + "ok": false + }, + { + "idx": 965, + "ok": false + }, + { + "idx": 800, + "ok": false + }, + { + "idx": 20, + "ok": false + }, + { + "idx": 1078, + "ok": false + }, + { + "idx": 136, + "ok": false + }, + { + "idx": 166, + "ok": true + }, + { + "idx": 809, + "ok": true + }, + { + "idx": 13, + "ok": false + }, + { + "idx": 739, + "ok": false + }, + { + "idx": 84, + "ok": true + }, + { + "idx": 237, + "ok": true + }, + { + "idx": 1271, + "ok": false + }, + { + "idx": 7, + "ok": false + }, + { + "idx": 553, + "ok": false + }, + { + "idx": 1311, + "ok": false + }, + { + "idx": 599, + "ok": false + }, + { + "idx": 464, + "ok": false + }, + { + "idx": 288, + "ok": false + }, + { + "idx": 1173, + "ok": false + }, + { + "idx": 589, + "ok": false + }, + { + "idx": 391, + "ok": false + }, + { + "idx": 888, + "ok": true + }, + { + "idx": 943, + "ok": false + }, + { + "idx": 676, + "ok": false + }, + { + "idx": 786, + "ok": false + }, + { + "idx": 344, + "ok": true + }, + { + "idx": 677, + "ok": false + }, + { + "idx": 863, + "ok": false + }, + { + "idx": 891, + "ok": false + }, + { + "idx": 303, + "ok": false + }, + { + "idx": 916, + "ok": false + }, + { + "idx": 302, + "ok": false + }, + { + "idx": 1073, + "ok": false + }, + { + "idx": 647, + "ok": false + }, + { + "idx": 264, + "ok": false + }, + { + "idx": 427, + "ok": false + }, + { + "idx": 382, + "ok": false + }, + { + "idx": 715, + "ok": false + }, + { + "idx": 796, + "ok": false + }, + { + "idx": 875, + "ok": false + }, + { + "idx": 1007, + "ok": false + }, + { + "idx": 797, + "ok": true + }, + { + "idx": 450, + "ok": false + }, + { + "idx": 401, + "ok": false + }, + { + "idx": 899, + "ok": false + }, + { + "idx": 418, + "ok": false + }, + { + "idx": 1201, + "ok": false + }, + { + "idx": 101, + "ok": false + }, + { + "idx": 795, + "ok": false + }, + { + "idx": 68, + "ok": false + }, + { + "idx": 479, + "ok": false + }, + { + "idx": 1296, + "ok": false + }, + { + "idx": 173, + "ok": false + }, + { + "idx": 744, + "ok": false + }, + { + "idx": 116, + "ok": true + }, + { + "idx": 1306, + "ok": false + }, + { + "idx": 355, + "ok": false + }, + { + "idx": 477, + "ok": false + }, + { + "idx": 1250, + "ok": false + }, + { + "idx": 609, + "ok": false + }, + { + "idx": 1254, + "ok": true + }, + { + "idx": 177, + "ok": false + }, + { + "idx": 1048, + "ok": false + }, + { + "idx": 582, + "ok": false + }, + { + "idx": 723, + "ok": false + }, + { + "idx": 842, + "ok": false + }, + { + "idx": 938, + "ok": false + }, + { + "idx": 110, + "ok": false + }, + { + "idx": 1057, + "ok": false + }, + { + "idx": 1120, + "ok": false + }, + { + "idx": 881, + "ok": false + }, + { + "idx": 1189, + "ok": false + }, + { + "idx": 931, + "ok": false + }, + { + "idx": 1003, + "ok": false + }, + { + "idx": 521, + "ok": false + }, + { + "idx": 974, + "ok": false + }, + { + "idx": 690, + "ok": false + }, + { + "idx": 544, + "ok": false + }, + { + "idx": 86, + "ok": false + }, + { + "idx": 89, + "ok": false + }, + { + "idx": 333, + "ok": false + }, + { + "idx": 716, + "ok": false + }, + { + "idx": 14, + "ok": false + }, + { + "idx": 287, + "ok": false + }, + { + "idx": 130, + "ok": false + }, + { + "idx": 876, + "ok": false + }, + { + "idx": 454, + "ok": false + }, + { + "idx": 1246, + "ok": true + }, + { + "idx": 812, + "ok": false + }, + { + "idx": 1142, + "ok": false + }, + { + "idx": 452, + "ok": false + }, + { + "idx": 929, + "ok": false + }, + { + "idx": 394, + "ok": false + }, + { + "idx": 695, + "ok": false + }, + { + "idx": 1245, + "ok": false + }, + { + "idx": 209, + "ok": false + }, + { + "idx": 1241, + "ok": false + }, + { + "idx": 175, + "ok": false + }, + { + "idx": 653, + "ok": false + }, + { + "idx": 661, + "ok": false + }, + { + "idx": 1097, + "ok": false + }, + { + "idx": 933, + "ok": false + }, + { + "idx": 665, + "ok": false + }, + { + "idx": 523, + "ok": false + }, + { + "idx": 59, + "ok": false + }, + { + "idx": 1069, + "ok": false + }, + { + "idx": 90, + "ok": false + }, + { + "idx": 389, + "ok": false + }, + { + "idx": 755, + "ok": false + }, + { + "idx": 164, + "ok": true + }, + { + "idx": 429, + "ok": false + }, + { + "idx": 1074, + "ok": false + }, + { + "idx": 708, + "ok": true + }, + { + "idx": 384, + "ok": false + }, + { + "idx": 412, + "ok": false + }, + { + "idx": 514, + "ok": false + }, + { + "idx": 617, + "ok": false + }, + { + "idx": 639, + "ok": false + }, + { + "idx": 1058, + "ok": false + }, + { + "idx": 787, + "ok": false + }, + { + "idx": 987, + "ok": false + }, + { + "idx": 704, + "ok": false + }, + { + "idx": 491, + "ok": false + }, + { + "idx": 91, + "ok": false + }, + { + "idx": 626, + "ok": false + }, + { + "idx": 1130, + "ok": false + }, + { + "idx": 148, + "ok": false + }, + { + "idx": 18, + "ok": false + }, + { + "idx": 897, + "ok": true + }, + { + "idx": 97, + "ok": false + }, + { + "idx": 844, + "ok": false + }, + { + "idx": 1010, + "ok": false + }, + { + "idx": 901, + "ok": false + }, + { + "idx": 241, + "ok": false + }, + { + "idx": 493, + "ok": false + }, + { + "idx": 202, + "ok": true + }, + { + "idx": 314, + "ok": false + }, + { + "idx": 847, + "ok": false + }, + { + "idx": 437, + "ok": false + }, + { + "idx": 902, + "ok": true + }, + { + "idx": 157, + "ok": false + }, + { + "idx": 874, + "ok": true + }, + { + "idx": 1144, + "ok": false + }, + { + "idx": 807, + "ok": false + }, + { + "idx": 80, + "ok": true + }, + { + "idx": 369, + "ok": false + }, + { + "idx": 511, + "ok": false + }, + { + "idx": 262, + "ok": false + }, + { + "idx": 571, + "ok": false + }, + { + "idx": 721, + "ok": false + }, + { + "idx": 654, + "ok": false + }, + { + "idx": 890, + "ok": true + }, + { + "idx": 219, + "ok": false + }, + { + "idx": 1140, + "ok": false + }, + { + "idx": 584, + "ok": false + }, + { + "idx": 1249, + "ok": false + }, + { + "idx": 1108, + "ok": true + }, + { + "idx": 607, + "ok": false + }, + { + "idx": 904, + "ok": false + }, + { + "idx": 1052, + "ok": false + }, + { + "idx": 1238, + "ok": false + }, + { + "idx": 946, + "ok": false + }, + { + "idx": 1098, + "ok": false + }, + { + "idx": 534, + "ok": false + }, + { + "idx": 558, + "ok": false + }, + { + "idx": 474, + "ok": true + }, + { + "idx": 33, + "ok": false + }, + { + "idx": 243, + "ok": false + }, + { + "idx": 1259, + "ok": false + }, + { + "idx": 353, + "ok": false + }, + { + "idx": 509, + "ok": false + }, + { + "idx": 1096, + "ok": false + }, + { + "idx": 1055, + "ok": false + }, + { + "idx": 877, + "ok": false + }, + { + "idx": 249, + "ok": false + }, + { + "idx": 1156, + "ok": false + }, + { + "idx": 735, + "ok": false + }, + { + "idx": 470, + "ok": false + }, + { + "idx": 1119, + "ok": false + }, + { + "idx": 577, + "ok": true + }, + { + "idx": 453, + "ok": false + }, + { + "idx": 132, + "ok": false + }, + { + "idx": 1061, + "ok": false + }, + { + "idx": 629, + "ok": false + }, + { + "idx": 670, + "ok": true + }, + { + "idx": 478, + "ok": false + }, + { + "idx": 764, + "ok": false + }, + { + "idx": 983, + "ok": false + }, + { + "idx": 587, + "ok": false + }, + { + "idx": 1192, + "ok": false + }, + { + "idx": 350, + "ok": false + }, + { + "idx": 281, + "ok": false + }, + { + "idx": 31, + "ok": false + }, + { + "idx": 1132, + "ok": false + }, + { + "idx": 1033, + "ok": false + }, + { + "idx": 671, + "ok": false + }, + { + "idx": 751, + "ok": false + } + ] + }, + "2": { + "acc": 0.0625, + "by_label": { + "drop": 0.04, + "easy": 0.3448275862068966, + "hard": 0.015748031496062992 + }, + "per_item": [ + { + "idx": 1092, + "ok": false + }, + { + "idx": 696, + "ok": false + }, + { + "idx": 242, + "ok": false + }, + { + "idx": 1085, + "ok": false + }, + { + "idx": 510, + "ok": false + }, + { + "idx": 334, + "ok": false + }, + { + "idx": 139, + "ok": false + }, + { + "idx": 849, + "ok": false + }, + { + "idx": 593, + "ok": false + }, + { + "idx": 579, + "ok": false + }, + { + "idx": 1064, + "ok": false + }, + { + "idx": 275, + "ok": false + }, + { + "idx": 1175, + "ok": false + }, + { + "idx": 1071, + "ok": false + }, + { + "idx": 1284, + "ok": false + }, + { + "idx": 430, + "ok": false + }, + { + "idx": 1088, + "ok": false + }, + { + "idx": 215, + "ok": false + }, + { + "idx": 841, + "ok": false + }, + { + "idx": 1299, + "ok": false + }, + { + "idx": 1113, + "ok": true + }, + { + "idx": 826, + "ok": false + }, + { + "idx": 570, + "ok": false + }, + { + "idx": 598, + "ok": false + }, + { + "idx": 905, + "ok": false + }, + { + "idx": 761, + "ok": false + }, + { + "idx": 1163, + "ok": false + }, + { + "idx": 1287, + "ok": false + }, + { + "idx": 282, + "ok": false + }, + { + "idx": 321, + "ok": false + }, + { + "idx": 252, + "ok": false + }, + { + "idx": 246, + "ok": false + }, + { + "idx": 781, + "ok": false + }, + { + "idx": 821, + "ok": false + }, + { + "idx": 1210, + "ok": true + }, + { + "idx": 958, + "ok": true + }, + { + "idx": 285, + "ok": false + }, + { + "idx": 1146, + "ok": false + }, + { + "idx": 612, + "ok": false + }, + { + "idx": 725, + "ok": false + }, + { + "idx": 1294, + "ok": false + }, + { + "idx": 968, + "ok": false + }, + { + "idx": 850, + "ok": false + }, + { + "idx": 446, + "ok": false + }, + { + "idx": 976, + "ok": false + }, + { + "idx": 1001, + "ok": false + }, + { + "idx": 1027, + "ok": false + }, + { + "idx": 651, + "ok": false + }, + { + "idx": 1009, + "ok": false + }, + { + "idx": 123, + "ok": false + }, + { + "idx": 909, + "ok": false + }, + { + "idx": 614, + "ok": false + }, + { + "idx": 292, + "ok": false + }, + { + "idx": 1014, + "ok": false + }, + { + "idx": 107, + "ok": true + }, + { + "idx": 1272, + "ok": false + }, + { + "idx": 441, + "ok": false + }, + { + "idx": 52, + "ok": false + }, + { + "idx": 727, + "ok": false + }, + { + "idx": 965, + "ok": false + }, + { + "idx": 800, + "ok": false + }, + { + "idx": 20, + "ok": false + }, + { + "idx": 1078, + "ok": false + }, + { + "idx": 136, + "ok": false + }, + { + "idx": 166, + "ok": false + }, + { + "idx": 809, + "ok": false + }, + { + "idx": 13, + "ok": false + }, + { + "idx": 739, + "ok": false + }, + { + "idx": 84, + "ok": false + }, + { + "idx": 237, + "ok": false + }, + { + "idx": 1271, + "ok": false + }, + { + "idx": 7, + "ok": false + }, + { + "idx": 553, + "ok": false + }, + { + "idx": 1311, + "ok": false + }, + { + "idx": 599, + "ok": false + }, + { + "idx": 464, + "ok": false + }, + { + "idx": 288, + "ok": false + }, + { + "idx": 1173, + "ok": false + }, + { + "idx": 589, + "ok": false + }, + { + "idx": 391, + "ok": false + }, + { + "idx": 888, + "ok": true + }, + { + "idx": 943, + "ok": false + }, + { + "idx": 676, + "ok": false + }, + { + "idx": 786, + "ok": false + }, + { + "idx": 344, + "ok": true + }, + { + "idx": 677, + "ok": false + }, + { + "idx": 863, + "ok": false + }, + { + "idx": 891, + "ok": false + }, + { + "idx": 303, + "ok": false + }, + { + "idx": 916, + "ok": false + }, + { + "idx": 302, + "ok": false + }, + { + "idx": 1073, + "ok": false + }, + { + "idx": 647, + "ok": false + }, + { + "idx": 264, + "ok": false + }, + { + "idx": 427, + "ok": false + }, + { + "idx": 382, + "ok": false + }, + { + "idx": 715, + "ok": false + }, + { + "idx": 796, + "ok": false + }, + { + "idx": 875, + "ok": false + }, + { + "idx": 1007, + "ok": false + }, + { + "idx": 797, + "ok": false + }, + { + "idx": 450, + "ok": false + }, + { + "idx": 401, + "ok": false + }, + { + "idx": 899, + "ok": false + }, + { + "idx": 418, + "ok": false + }, + { + "idx": 1201, + "ok": false + }, + { + "idx": 101, + "ok": false + }, + { + "idx": 795, + "ok": false + }, + { + "idx": 68, + "ok": false + }, + { + "idx": 479, + "ok": false + }, + { + "idx": 1296, + "ok": false + }, + { + "idx": 173, + "ok": false + }, + { + "idx": 744, + "ok": false + }, + { + "idx": 116, + "ok": false + }, + { + "idx": 1306, + "ok": false + }, + { + "idx": 355, + "ok": false + }, + { + "idx": 477, + "ok": false + }, + { + "idx": 1250, + "ok": false + }, + { + "idx": 609, + "ok": false + }, + { + "idx": 1254, + "ok": false + }, + { + "idx": 177, + "ok": false + }, + { + "idx": 1048, + "ok": false + }, + { + "idx": 582, + "ok": false + }, + { + "idx": 723, + "ok": false + }, + { + "idx": 842, + "ok": true + }, + { + "idx": 938, + "ok": false + }, + { + "idx": 110, + "ok": false + }, + { + "idx": 1057, + "ok": false + }, + { + "idx": 1120, + "ok": false + }, + { + "idx": 881, + "ok": false + }, + { + "idx": 1189, + "ok": false + }, + { + "idx": 931, + "ok": false + }, + { + "idx": 1003, + "ok": false + }, + { + "idx": 521, + "ok": false + }, + { + "idx": 974, + "ok": false + }, + { + "idx": 690, + "ok": false + }, + { + "idx": 544, + "ok": false + }, + { + "idx": 86, + "ok": false + }, + { + "idx": 89, + "ok": true + }, + { + "idx": 333, + "ok": false + }, + { + "idx": 716, + "ok": false + }, + { + "idx": 14, + "ok": false + }, + { + "idx": 287, + "ok": false + }, + { + "idx": 130, + "ok": false + }, + { + "idx": 876, + "ok": false + }, + { + "idx": 454, + "ok": false + }, + { + "idx": 1246, + "ok": false + }, + { + "idx": 812, + "ok": false + }, + { + "idx": 1142, + "ok": false + }, + { + "idx": 452, + "ok": false + }, + { + "idx": 929, + "ok": false + }, + { + "idx": 394, + "ok": false + }, + { + "idx": 695, + "ok": false + }, + { + "idx": 1245, + "ok": false + }, + { + "idx": 209, + "ok": false + }, + { + "idx": 1241, + "ok": false + }, + { + "idx": 175, + "ok": false + }, + { + "idx": 653, + "ok": false + }, + { + "idx": 661, + "ok": false + }, + { + "idx": 1097, + "ok": false + }, + { + "idx": 933, + "ok": false + }, + { + "idx": 665, + "ok": false + }, + { + "idx": 523, + "ok": false + }, + { + "idx": 59, + "ok": false + }, + { + "idx": 1069, + "ok": false + }, + { + "idx": 90, + "ok": false + }, + { + "idx": 389, + "ok": false + }, + { + "idx": 755, + "ok": false + }, + { + "idx": 164, + "ok": false + }, + { + "idx": 429, + "ok": false + }, + { + "idx": 1074, + "ok": false + }, + { + "idx": 708, + "ok": true + }, + { + "idx": 384, + "ok": false + }, + { + "idx": 412, + "ok": false + }, + { + "idx": 514, + "ok": false + }, + { + "idx": 617, + "ok": false + }, + { + "idx": 639, + "ok": false + }, + { + "idx": 1058, + "ok": false + }, + { + "idx": 787, + "ok": false + }, + { + "idx": 987, + "ok": false + }, + { + "idx": 704, + "ok": false + }, + { + "idx": 491, + "ok": false + }, + { + "idx": 91, + "ok": false + }, + { + "idx": 626, + "ok": false + }, + { + "idx": 1130, + "ok": false + }, + { + "idx": 148, + "ok": false + }, + { + "idx": 18, + "ok": false + }, + { + "idx": 897, + "ok": false + }, + { + "idx": 97, + "ok": false + }, + { + "idx": 844, + "ok": false + }, + { + "idx": 1010, + "ok": false + }, + { + "idx": 901, + "ok": true + }, + { + "idx": 241, + "ok": true + }, + { + "idx": 493, + "ok": false + }, + { + "idx": 202, + "ok": true + }, + { + "idx": 314, + "ok": false + }, + { + "idx": 847, + "ok": false + }, + { + "idx": 437, + "ok": false + }, + { + "idx": 902, + "ok": true + }, + { + "idx": 157, + "ok": false + }, + { + "idx": 874, + "ok": true + }, + { + "idx": 1144, + "ok": false + }, + { + "idx": 807, + "ok": false + }, + { + "idx": 80, + "ok": false + }, + { + "idx": 369, + "ok": false + }, + { + "idx": 511, + "ok": false + }, + { + "idx": 262, + "ok": false + }, + { + "idx": 571, + "ok": false + }, + { + "idx": 721, + "ok": false + }, + { + "idx": 654, + "ok": false + }, + { + "idx": 890, + "ok": false + }, + { + "idx": 219, + "ok": false + }, + { + "idx": 1140, + "ok": false + }, + { + "idx": 584, + "ok": true + }, + { + "idx": 1249, + "ok": false + }, + { + "idx": 1108, + "ok": true + }, + { + "idx": 607, + "ok": false + }, + { + "idx": 904, + "ok": false + }, + { + "idx": 1052, + "ok": false + }, + { + "idx": 1238, + "ok": false + }, + { + "idx": 946, + "ok": false + }, + { + "idx": 1098, + "ok": false + }, + { + "idx": 534, + "ok": false + }, + { + "idx": 558, + "ok": false + }, + { + "idx": 474, + "ok": false + }, + { + "idx": 33, + "ok": false + }, + { + "idx": 243, + "ok": false + }, + { + "idx": 1259, + "ok": false + }, + { + "idx": 353, + "ok": false + }, + { + "idx": 509, + "ok": false + }, + { + "idx": 1096, + "ok": false + }, + { + "idx": 1055, + "ok": false + }, + { + "idx": 877, + "ok": false + }, + { + "idx": 249, + "ok": false + }, + { + "idx": 1156, + "ok": false + }, + { + "idx": 735, + "ok": false + }, + { + "idx": 470, + "ok": false + }, + { + "idx": 1119, + "ok": false + }, + { + "idx": 577, + "ok": false + }, + { + "idx": 453, + "ok": false + }, + { + "idx": 132, + "ok": false + }, + { + "idx": 1061, + "ok": false + }, + { + "idx": 629, + "ok": false + }, + { + "idx": 670, + "ok": false + }, + { + "idx": 478, + "ok": false + }, + { + "idx": 764, + "ok": false + }, + { + "idx": 983, + "ok": false + }, + { + "idx": 587, + "ok": false + }, + { + "idx": 1192, + "ok": false + }, + { + "idx": 350, + "ok": false + }, + { + "idx": 281, + "ok": false + }, + { + "idx": 31, + "ok": false + }, + { + "idx": 1132, + "ok": false + }, + { + "idx": 1033, + "ok": false + }, + { + "idx": 671, + "ok": false + }, + { + "idx": 751, + "ok": false + } + ] + }, + "4": { + "acc": 0.0625, + "by_label": { + "drop": 0.04, + "easy": 0.3448275862068966, + "hard": 0.015748031496062992 + }, + "per_item": [ + { + "idx": 1092, + "ok": false + }, + { + "idx": 696, + "ok": false + }, + { + "idx": 242, + "ok": false + }, + { + "idx": 1085, + "ok": false + }, + { + "idx": 510, + "ok": false + }, + { + "idx": 334, + "ok": false + }, + { + "idx": 139, + "ok": false + }, + { + "idx": 849, + "ok": false + }, + { + "idx": 593, + "ok": false + }, + { + "idx": 579, + "ok": false + }, + { + "idx": 1064, + "ok": false + }, + { + "idx": 275, + "ok": false + }, + { + "idx": 1175, + "ok": false + }, + { + "idx": 1071, + "ok": false + }, + { + "idx": 1284, + "ok": false + }, + { + "idx": 430, + "ok": false + }, + { + "idx": 1088, + "ok": false + }, + { + "idx": 215, + "ok": false + }, + { + "idx": 841, + "ok": false + }, + { + "idx": 1299, + "ok": false + }, + { + "idx": 1113, + "ok": true + }, + { + "idx": 826, + "ok": false + }, + { + "idx": 570, + "ok": false + }, + { + "idx": 598, + "ok": false + }, + { + "idx": 905, + "ok": false + }, + { + "idx": 761, + "ok": false + }, + { + "idx": 1163, + "ok": false + }, + { + "idx": 1287, + "ok": false + }, + { + "idx": 282, + "ok": false + }, + { + "idx": 321, + "ok": false + }, + { + "idx": 252, + "ok": false + }, + { + "idx": 246, + "ok": false + }, + { + "idx": 781, + "ok": false + }, + { + "idx": 821, + "ok": false + }, + { + "idx": 1210, + "ok": true + }, + { + "idx": 958, + "ok": true + }, + { + "idx": 285, + "ok": false + }, + { + "idx": 1146, + "ok": false + }, + { + "idx": 612, + "ok": false + }, + { + "idx": 725, + "ok": false + }, + { + "idx": 1294, + "ok": false + }, + { + "idx": 968, + "ok": false + }, + { + "idx": 850, + "ok": false + }, + { + "idx": 446, + "ok": false + }, + { + "idx": 976, + "ok": false + }, + { + "idx": 1001, + "ok": false + }, + { + "idx": 1027, + "ok": false + }, + { + "idx": 651, + "ok": false + }, + { + "idx": 1009, + "ok": false + }, + { + "idx": 123, + "ok": false + }, + { + "idx": 909, + "ok": false + }, + { + "idx": 614, + "ok": false + }, + { + "idx": 292, + "ok": false + }, + { + "idx": 1014, + "ok": false + }, + { + "idx": 107, + "ok": true + }, + { + "idx": 1272, + "ok": false + }, + { + "idx": 441, + "ok": false + }, + { + "idx": 52, + "ok": false + }, + { + "idx": 727, + "ok": false + }, + { + "idx": 965, + "ok": false + }, + { + "idx": 800, + "ok": false + }, + { + "idx": 20, + "ok": false + }, + { + "idx": 1078, + "ok": false + }, + { + "idx": 136, + "ok": false + }, + { + "idx": 166, + "ok": false + }, + { + "idx": 809, + "ok": false + }, + { + "idx": 13, + "ok": false + }, + { + "idx": 739, + "ok": false + }, + { + "idx": 84, + "ok": false + }, + { + "idx": 237, + "ok": false + }, + { + "idx": 1271, + "ok": false + }, + { + "idx": 7, + "ok": false + }, + { + "idx": 553, + "ok": false + }, + { + "idx": 1311, + "ok": false + }, + { + "idx": 599, + "ok": false + }, + { + "idx": 464, + "ok": false + }, + { + "idx": 288, + "ok": false + }, + { + "idx": 1173, + "ok": false + }, + { + "idx": 589, + "ok": false + }, + { + "idx": 391, + "ok": false + }, + { + "idx": 888, + "ok": true + }, + { + "idx": 943, + "ok": false + }, + { + "idx": 676, + "ok": false + }, + { + "idx": 786, + "ok": false + }, + { + "idx": 344, + "ok": true + }, + { + "idx": 677, + "ok": false + }, + { + "idx": 863, + "ok": false + }, + { + "idx": 891, + "ok": false + }, + { + "idx": 303, + "ok": false + }, + { + "idx": 916, + "ok": false + }, + { + "idx": 302, + "ok": false + }, + { + "idx": 1073, + "ok": false + }, + { + "idx": 647, + "ok": false + }, + { + "idx": 264, + "ok": false + }, + { + "idx": 427, + "ok": false + }, + { + "idx": 382, + "ok": false + }, + { + "idx": 715, + "ok": false + }, + { + "idx": 796, + "ok": false + }, + { + "idx": 875, + "ok": false + }, + { + "idx": 1007, + "ok": false + }, + { + "idx": 797, + "ok": false + }, + { + "idx": 450, + "ok": false + }, + { + "idx": 401, + "ok": false + }, + { + "idx": 899, + "ok": false + }, + { + "idx": 418, + "ok": false + }, + { + "idx": 1201, + "ok": false + }, + { + "idx": 101, + "ok": false + }, + { + "idx": 795, + "ok": false + }, + { + "idx": 68, + "ok": false + }, + { + "idx": 479, + "ok": false + }, + { + "idx": 1296, + "ok": false + }, + { + "idx": 173, + "ok": false + }, + { + "idx": 744, + "ok": false + }, + { + "idx": 116, + "ok": false + }, + { + "idx": 1306, + "ok": false + }, + { + "idx": 355, + "ok": false + }, + { + "idx": 477, + "ok": false + }, + { + "idx": 1250, + "ok": false + }, + { + "idx": 609, + "ok": false + }, + { + "idx": 1254, + "ok": false + }, + { + "idx": 177, + "ok": false + }, + { + "idx": 1048, + "ok": false + }, + { + "idx": 582, + "ok": false + }, + { + "idx": 723, + "ok": false + }, + { + "idx": 842, + "ok": true + }, + { + "idx": 938, + "ok": false + }, + { + "idx": 110, + "ok": false + }, + { + "idx": 1057, + "ok": false + }, + { + "idx": 1120, + "ok": false + }, + { + "idx": 881, + "ok": false + }, + { + "idx": 1189, + "ok": false + }, + { + "idx": 931, + "ok": false + }, + { + "idx": 1003, + "ok": false + }, + { + "idx": 521, + "ok": false + }, + { + "idx": 974, + "ok": false + }, + { + "idx": 690, + "ok": false + }, + { + "idx": 544, + "ok": false + }, + { + "idx": 86, + "ok": false + }, + { + "idx": 89, + "ok": true + }, + { + "idx": 333, + "ok": false + }, + { + "idx": 716, + "ok": false + }, + { + "idx": 14, + "ok": false + }, + { + "idx": 287, + "ok": false + }, + { + "idx": 130, + "ok": false + }, + { + "idx": 876, + "ok": false + }, + { + "idx": 454, + "ok": false + }, + { + "idx": 1246, + "ok": false + }, + { + "idx": 812, + "ok": false + }, + { + "idx": 1142, + "ok": false + }, + { + "idx": 452, + "ok": false + }, + { + "idx": 929, + "ok": false + }, + { + "idx": 394, + "ok": false + }, + { + "idx": 695, + "ok": false + }, + { + "idx": 1245, + "ok": false + }, + { + "idx": 209, + "ok": false + }, + { + "idx": 1241, + "ok": false + }, + { + "idx": 175, + "ok": false + }, + { + "idx": 653, + "ok": false + }, + { + "idx": 661, + "ok": false + }, + { + "idx": 1097, + "ok": false + }, + { + "idx": 933, + "ok": false + }, + { + "idx": 665, + "ok": false + }, + { + "idx": 523, + "ok": false + }, + { + "idx": 59, + "ok": false + }, + { + "idx": 1069, + "ok": false + }, + { + "idx": 90, + "ok": false + }, + { + "idx": 389, + "ok": false + }, + { + "idx": 755, + "ok": false + }, + { + "idx": 164, + "ok": false + }, + { + "idx": 429, + "ok": false + }, + { + "idx": 1074, + "ok": false + }, + { + "idx": 708, + "ok": true + }, + { + "idx": 384, + "ok": false + }, + { + "idx": 412, + "ok": false + }, + { + "idx": 514, + "ok": false + }, + { + "idx": 617, + "ok": false + }, + { + "idx": 639, + "ok": false + }, + { + "idx": 1058, + "ok": false + }, + { + "idx": 787, + "ok": false + }, + { + "idx": 987, + "ok": false + }, + { + "idx": 704, + "ok": false + }, + { + "idx": 491, + "ok": false + }, + { + "idx": 91, + "ok": false + }, + { + "idx": 626, + "ok": false + }, + { + "idx": 1130, + "ok": false + }, + { + "idx": 148, + "ok": false + }, + { + "idx": 18, + "ok": false + }, + { + "idx": 897, + "ok": false + }, + { + "idx": 97, + "ok": false + }, + { + "idx": 844, + "ok": false + }, + { + "idx": 1010, + "ok": false + }, + { + "idx": 901, + "ok": true + }, + { + "idx": 241, + "ok": true + }, + { + "idx": 493, + "ok": false + }, + { + "idx": 202, + "ok": true + }, + { + "idx": 314, + "ok": false + }, + { + "idx": 847, + "ok": false + }, + { + "idx": 437, + "ok": false + }, + { + "idx": 902, + "ok": true + }, + { + "idx": 157, + "ok": false + }, + { + "idx": 874, + "ok": true + }, + { + "idx": 1144, + "ok": false + }, + { + "idx": 807, + "ok": false + }, + { + "idx": 80, + "ok": false + }, + { + "idx": 369, + "ok": false + }, + { + "idx": 511, + "ok": false + }, + { + "idx": 262, + "ok": false + }, + { + "idx": 571, + "ok": false + }, + { + "idx": 721, + "ok": false + }, + { + "idx": 654, + "ok": false + }, + { + "idx": 890, + "ok": false + }, + { + "idx": 219, + "ok": false + }, + { + "idx": 1140, + "ok": false + }, + { + "idx": 584, + "ok": true + }, + { + "idx": 1249, + "ok": false + }, + { + "idx": 1108, + "ok": true + }, + { + "idx": 607, + "ok": false + }, + { + "idx": 904, + "ok": false + }, + { + "idx": 1052, + "ok": false + }, + { + "idx": 1238, + "ok": false + }, + { + "idx": 946, + "ok": false + }, + { + "idx": 1098, + "ok": false + }, + { + "idx": 534, + "ok": false + }, + { + "idx": 558, + "ok": false + }, + { + "idx": 474, + "ok": false + }, + { + "idx": 33, + "ok": false + }, + { + "idx": 243, + "ok": false + }, + { + "idx": 1259, + "ok": false + }, + { + "idx": 353, + "ok": false + }, + { + "idx": 509, + "ok": false + }, + { + "idx": 1096, + "ok": false + }, + { + "idx": 1055, + "ok": false + }, + { + "idx": 877, + "ok": false + }, + { + "idx": 249, + "ok": false + }, + { + "idx": 1156, + "ok": false + }, + { + "idx": 735, + "ok": false + }, + { + "idx": 470, + "ok": false + }, + { + "idx": 1119, + "ok": false + }, + { + "idx": 577, + "ok": false + }, + { + "idx": 453, + "ok": false + }, + { + "idx": 132, + "ok": false + }, + { + "idx": 1061, + "ok": false + }, + { + "idx": 629, + "ok": false + }, + { + "idx": 670, + "ok": false + }, + { + "idx": 478, + "ok": false + }, + { + "idx": 764, + "ok": false + }, + { + "idx": 983, + "ok": false + }, + { + "idx": 587, + "ok": false + }, + { + "idx": 1192, + "ok": false + }, + { + "idx": 350, + "ok": false + }, + { + "idx": 281, + "ok": false + }, + { + "idx": 31, + "ok": false + }, + { + "idx": 1132, + "ok": false + }, + { + "idx": 1033, + "ok": false + }, + { + "idx": 671, + "ok": false + }, + { + "idx": 751, + "ok": false + } + ] + } + }, + "n": 256, + "spider": [] +} \ No newline at end of file diff --git a/results-loop/eval_code_merge_ns_s1_e400.json b/results-loop/eval_code_merge_ns_s1_e400.json new file mode 100644 index 0000000..d5566a0 --- /dev/null +++ b/results-loop/eval_code_merge_ns_s1_e400.json @@ -0,0 +1,3036 @@ +{ + "tag": "merge_ns_s1_e400", + "ks": { + "0": { + "acc": 0.488, + "by_label": { + "easy": 0.9836065573770492, + "hard": 0.03571428571428571, + "drop": 0.01 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": false + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": true + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": true + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": false + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": false + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": false + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": false + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": false + }, + { + "task_id": 131, + "ok": true + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": true + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": false + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": false + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": false + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": true + }, + { + "task_id": 195, + "ok": false + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": false + }, + { + "task_id": 213, + "ok": true + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": true + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": false + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": true + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": true + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": false + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + } + ] + }, + "2": { + "acc": 0.508, + "by_label": { + "easy": 0.9426229508196722, + "hard": 0.25, + "drop": 0.05 + }, + "per_item": [ + { + "task_id": 11, + "ok": false + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": true + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": true + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": true + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": true + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": true + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": true + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": true + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": false + }, + { + "task_id": 131, + "ok": true + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": false + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": true + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": false + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": true + }, + { + "task_id": 166, + "ok": false + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": true + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": false + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": false + }, + { + "task_id": 195, + "ok": true + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": true + }, + { + "task_id": 213, + "ok": false + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": false + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": true + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": false + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": true + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + } + ] + }, + "4": { + "acc": 0.52, + "by_label": { + "easy": 0.9180327868852459, + "hard": 0.39285714285714285, + "drop": 0.07 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": true + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": true + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": false + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": false + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": true + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": true + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": true + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": true + }, + { + "task_id": 131, + "ok": true + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": false + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": true + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": false + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": true + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": false + }, + { + "task_id": 154, + "ok": false + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": true + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": true + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": false + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": true + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": false + }, + { + "task_id": 195, + "ok": true + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": false + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": true + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": true + }, + { + "task_id": 213, + "ok": false + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": true + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": true + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": true + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": false + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": true + }, + { + "task_id": 259, + "ok": true + }, + { + "task_id": 260, + "ok": false + } + ] + } + }, + "n": 250 +} \ No newline at end of file diff --git a/results-loop/eval_code_merge_ns_s2_e400.json b/results-loop/eval_code_merge_ns_s2_e400.json new file mode 100644 index 0000000..dd81181 --- /dev/null +++ b/results-loop/eval_code_merge_ns_s2_e400.json @@ -0,0 +1,3036 @@ +{ + "tag": "merge_ns_s2_e400", + "ks": { + "0": { + "acc": 0.488, + "by_label": { + "easy": 0.9836065573770492, + "hard": 0.03571428571428571, + "drop": 0.01 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": false + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": true + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": true + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": false + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": false + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": false + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": false + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": false + }, + { + "task_id": 131, + "ok": true + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": true + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": false + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": false + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": false + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": true + }, + { + "task_id": 195, + "ok": false + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": false + }, + { + "task_id": 213, + "ok": true + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": true + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": false + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": true + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": true + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": false + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + } + ] + }, + "2": { + "acc": 0.488, + "by_label": { + "easy": 0.9016393442622951, + "hard": 0.17857142857142858, + "drop": 0.07 + }, + "per_item": [ + { + "task_id": 11, + "ok": false + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": false + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": true + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": true + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": false + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": false + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": false + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": false + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": false + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": true + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": false + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": true + }, + { + "task_id": 131, + "ok": true + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": false + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": true + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": true + }, + { + "task_id": 166, + "ok": false + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": true + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": false + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": true + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": true + }, + { + "task_id": 195, + "ok": true + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": true + }, + { + "task_id": 213, + "ok": true + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": false + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": true + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": false + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": false + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": true + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + } + ] + }, + "4": { + "acc": 0.508, + "by_label": { + "easy": 0.9180327868852459, + "hard": 0.35714285714285715, + "drop": 0.05 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": false + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": false + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": true + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": true + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": false + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": false + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": false + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": true + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": false + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": false + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": true + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": false + }, + { + "task_id": 71, + "ok": false + }, + { + "task_id": 72, + "ok": true + }, + { + "task_id": 73, + "ok": false + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": false + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": true + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": true + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": true + }, + { + "task_id": 118, + "ok": false + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": true + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": true + }, + { + "task_id": 131, + "ok": true + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": false + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": true + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": false + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": false + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": true + }, + { + "task_id": 167, + "ok": true + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": false + }, + { + "task_id": 170, + "ok": true + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": false + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": true + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": false + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": false + }, + { + "task_id": 194, + "ok": true + }, + { + "task_id": 195, + "ok": true + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": true + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": true + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": true + }, + { + "task_id": 213, + "ok": true + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": false + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": true + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": true + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": false + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": false + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": false + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": true + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + } + ] + } + }, + "n": 250 +} \ No newline at end of file