diff --git a/PAPER.md b/PAPER.md index 182fe9b..e0aa1b0 100644 --- a/PAPER.md +++ b/PAPER.md @@ -247,7 +247,10 @@ samples, so the comparison is exact), best-of-3 is statistically indistinguishable from the latent arms overall, *behind* them directionally on the hard bucket, and pays ā‰ˆ3Ɨ visible tokens and serial decode for it. Budget-CoT-50 remains the strongest honest token baseline -(53.8% overall, hard 40.0%). The implant's advantages at matched compute +(53.8% overall, hard 40.0%; per-item rerun 53.8/38.2) — and the paired +tests confirm it is a *tie* with the latent arms on both axes (p≄0.69 vs +loop and distill), at the price of 50 visible tokens and their serial +decode latency. The implant's advantages at matched compute without a verifier: zero visible tokens, zero decode overhead, and the hard-slice crown under distillation (46%). Where a task *does* come with a cheap verifier, oracle-style sampling is the better overall-accuracy diff --git a/results-loop/STATS.md b/results-loop/STATS.md index e991d6d..6c8e9d8 100644 --- a/results-loop/STATS.md +++ b/results-loop/STATS.md @@ -44,7 +44,11 @@ - bo3-deployable vs loop k=4, hard: arm-only 16, bo3-only 7, p=0.09314 (n.s.) - bo3-deployable vs distill k=1, overall: arm-only 31, bo3-only 34, p=0.8043 (n.s.) - bo3-deployable vs distill k=1, hard: arm-only 15, bo3-only 7, p=0.1338 (n.s.) -(token-baseline per-item files incomplete: [Errno 2] No such file or directory: '/home/nils/jspace/results-loop/eval_budgetcot_pi.json') +- **budget-CoT-50 (per-item rerun)**: overall 0.538 [0.494, 0.581] (n=500); hard 0.382 [0.265, 0.514] (n=55) +- budget-CoT vs loop k=4, overall: arm-only 36, cot-only 37, p=1 (n.s.) +- budget-CoT vs loop k=4, hard: arm-only 14, cot-only 11, p=0.69 (n.s.) +- budget-CoT vs distill k=1, overall: arm-only 29, cot-only 26, p=0.7877 (n.s.) +- budget-CoT vs distill k=1, hard: arm-only 8, cot-only 6, p=0.7905 (n.s.) ## Pooled hard bucket (MBPP + HumanEval + Rust) diff --git a/results-loop/eval_budgetcot_pi.json b/results-loop/eval_budgetcot_pi.json new file mode 100644 index 0000000..1cf63ad --- /dev/null +++ b/results-loop/eval_budgetcot_pi.json @@ -0,0 +1,2010 @@ +{ + "acc": 0.538, + "by_label": { + "easy": 0.9038461538461539, + "hard": 0.38181818181818183, + "drop": 0.07027027027027027 + }, + "per_item": [ + { + "task_id": 11, + "ok": true + }, + { + "task_id": 12, + "ok": true + }, + { + "task_id": 13, + "ok": false + }, + { + "task_id": 14, + "ok": true + }, + { + "task_id": 15, + "ok": false + }, + { + "task_id": 16, + "ok": false + }, + { + "task_id": 17, + "ok": true + }, + { + "task_id": 18, + "ok": true + }, + { + "task_id": 19, + "ok": true + }, + { + "task_id": 20, + "ok": true + }, + { + "task_id": 21, + "ok": false + }, + { + "task_id": 22, + "ok": true + }, + { + "task_id": 23, + "ok": true + }, + { + "task_id": 24, + "ok": false + }, + { + "task_id": 25, + "ok": true + }, + { + "task_id": 26, + "ok": false + }, + { + "task_id": 27, + "ok": true + }, + { + "task_id": 28, + "ok": false + }, + { + "task_id": 29, + "ok": true + }, + { + "task_id": 30, + "ok": true + }, + { + "task_id": 31, + "ok": false + }, + { + "task_id": 32, + "ok": false + }, + { + "task_id": 33, + "ok": false + }, + { + "task_id": 34, + "ok": false + }, + { + "task_id": 35, + "ok": false + }, + { + "task_id": 36, + "ok": false + }, + { + "task_id": 37, + "ok": true + }, + { + "task_id": 38, + "ok": false + }, + { + "task_id": 39, + "ok": false + }, + { + "task_id": 40, + "ok": true + }, + { + "task_id": 41, + "ok": true + }, + { + "task_id": 42, + "ok": false + }, + { + "task_id": 43, + "ok": false + }, + { + "task_id": 44, + "ok": false + }, + { + "task_id": 45, + "ok": true + }, + { + "task_id": 46, + "ok": true + }, + { + "task_id": 47, + "ok": false + }, + { + "task_id": 48, + "ok": false + }, + { + "task_id": 49, + "ok": true + }, + { + "task_id": 50, + "ok": true + }, + { + "task_id": 51, + "ok": true + }, + { + "task_id": 52, + "ok": true + }, + { + "task_id": 53, + "ok": true + }, + { + "task_id": 54, + "ok": false + }, + { + "task_id": 55, + "ok": true + }, + { + "task_id": 56, + "ok": true + }, + { + "task_id": 57, + "ok": true + }, + { + "task_id": 58, + "ok": true + }, + { + "task_id": 59, + "ok": true + }, + { + "task_id": 60, + "ok": false + }, + { + "task_id": 61, + "ok": true + }, + { + "task_id": 62, + "ok": true + }, + { + "task_id": 63, + "ok": false + }, + { + "task_id": 64, + "ok": true + }, + { + "task_id": 65, + "ok": true + }, + { + "task_id": 66, + "ok": true + }, + { + "task_id": 67, + "ok": false + }, + { + "task_id": 68, + "ok": true + }, + { + "task_id": 69, + "ok": true + }, + { + "task_id": 70, + "ok": true + }, + { + "task_id": 71, + "ok": true + }, + { + "task_id": 72, + "ok": false + }, + { + "task_id": 73, + "ok": true + }, + { + "task_id": 74, + "ok": false + }, + { + "task_id": 75, + "ok": true + }, + { + "task_id": 76, + "ok": true + }, + { + "task_id": 77, + "ok": false + }, + { + "task_id": 78, + "ok": true + }, + { + "task_id": 79, + "ok": true + }, + { + "task_id": 80, + "ok": true + }, + { + "task_id": 81, + "ok": false + }, + { + "task_id": 82, + "ok": true + }, + { + "task_id": 83, + "ok": false + }, + { + "task_id": 84, + "ok": false + }, + { + "task_id": 85, + "ok": true + }, + { + "task_id": 86, + "ok": true + }, + { + "task_id": 87, + "ok": false + }, + { + "task_id": 88, + "ok": true + }, + { + "task_id": 89, + "ok": true + }, + { + "task_id": 90, + "ok": true + }, + { + "task_id": 91, + "ok": true + }, + { + "task_id": 92, + "ok": false + }, + { + "task_id": 93, + "ok": true + }, + { + "task_id": 94, + "ok": true + }, + { + "task_id": 95, + "ok": true + }, + { + "task_id": 96, + "ok": true + }, + { + "task_id": 97, + "ok": true + }, + { + "task_id": 98, + "ok": true + }, + { + "task_id": 99, + "ok": true + }, + { + "task_id": 100, + "ok": false + }, + { + "task_id": 101, + "ok": false + }, + { + "task_id": 102, + "ok": true + }, + { + "task_id": 103, + "ok": false + }, + { + "task_id": 104, + "ok": true + }, + { + "task_id": 105, + "ok": true + }, + { + "task_id": 106, + "ok": false + }, + { + "task_id": 107, + "ok": false + }, + { + "task_id": 108, + "ok": false + }, + { + "task_id": 109, + "ok": false + }, + { + "task_id": 110, + "ok": false + }, + { + "task_id": 111, + "ok": false + }, + { + "task_id": 112, + "ok": false + }, + { + "task_id": 113, + "ok": true + }, + { + "task_id": 114, + "ok": false + }, + { + "task_id": 115, + "ok": true + }, + { + "task_id": 116, + "ok": true + }, + { + "task_id": 117, + "ok": false + }, + { + "task_id": 118, + "ok": true + }, + { + "task_id": 119, + "ok": false + }, + { + "task_id": 120, + "ok": false + }, + { + "task_id": 121, + "ok": false + }, + { + "task_id": 122, + "ok": false + }, + { + "task_id": 123, + "ok": false + }, + { + "task_id": 124, + "ok": false + }, + { + "task_id": 125, + "ok": false + }, + { + "task_id": 126, + "ok": false + }, + { + "task_id": 127, + "ok": true + }, + { + "task_id": 128, + "ok": true + }, + { + "task_id": 129, + "ok": false + }, + { + "task_id": 130, + "ok": false + }, + { + "task_id": 131, + "ok": true + }, + { + "task_id": 132, + "ok": true + }, + { + "task_id": 133, + "ok": true + }, + { + "task_id": 134, + "ok": false + }, + { + "task_id": 135, + "ok": true + }, + { + "task_id": 136, + "ok": false + }, + { + "task_id": 137, + "ok": false + }, + { + "task_id": 138, + "ok": false + }, + { + "task_id": 139, + "ok": false + }, + { + "task_id": 140, + "ok": false + }, + { + "task_id": 141, + "ok": false + }, + { + "task_id": 142, + "ok": false + }, + { + "task_id": 143, + "ok": false + }, + { + "task_id": 144, + "ok": true + }, + { + "task_id": 145, + "ok": true + }, + { + "task_id": 146, + "ok": false + }, + { + "task_id": 147, + "ok": false + }, + { + "task_id": 148, + "ok": false + }, + { + "task_id": 149, + "ok": false + }, + { + "task_id": 150, + "ok": false + }, + { + "task_id": 151, + "ok": true + }, + { + "task_id": 152, + "ok": true + }, + { + "task_id": 153, + "ok": true + }, + { + "task_id": 154, + "ok": true + }, + { + "task_id": 155, + "ok": false + }, + { + "task_id": 156, + "ok": true + }, + { + "task_id": 157, + "ok": true + }, + { + "task_id": 158, + "ok": false + }, + { + "task_id": 159, + "ok": false + }, + { + "task_id": 160, + "ok": false + }, + { + "task_id": 161, + "ok": true + }, + { + "task_id": 162, + "ok": true + }, + { + "task_id": 163, + "ok": true + }, + { + "task_id": 164, + "ok": false + }, + { + "task_id": 165, + "ok": false + }, + { + "task_id": 166, + "ok": true + }, + { + "task_id": 167, + "ok": false + }, + { + "task_id": 168, + "ok": true + }, + { + "task_id": 169, + "ok": true + }, + { + "task_id": 170, + "ok": true + }, + { + "task_id": 171, + "ok": true + }, + { + "task_id": 172, + "ok": true + }, + { + "task_id": 173, + "ok": true + }, + { + "task_id": 174, + "ok": true + }, + { + "task_id": 175, + "ok": true + }, + { + "task_id": 176, + "ok": true + }, + { + "task_id": 177, + "ok": false + }, + { + "task_id": 178, + "ok": true + }, + { + "task_id": 179, + "ok": true + }, + { + "task_id": 180, + "ok": false + }, + { + "task_id": 181, + "ok": false + }, + { + "task_id": 182, + "ok": false + }, + { + "task_id": 183, + "ok": false + }, + { + "task_id": 184, + "ok": false + }, + { + "task_id": 185, + "ok": false + }, + { + "task_id": 186, + "ok": true + }, + { + "task_id": 187, + "ok": false + }, + { + "task_id": 188, + "ok": false + }, + { + "task_id": 189, + "ok": true + }, + { + "task_id": 190, + "ok": false + }, + { + "task_id": 191, + "ok": true + }, + { + "task_id": 192, + "ok": true + }, + { + "task_id": 193, + "ok": true + }, + { + "task_id": 194, + "ok": true + }, + { + "task_id": 195, + "ok": false + }, + { + "task_id": 196, + "ok": true + }, + { + "task_id": 197, + "ok": true + }, + { + "task_id": 198, + "ok": false + }, + { + "task_id": 199, + "ok": true + }, + { + "task_id": 200, + "ok": true + }, + { + "task_id": 201, + "ok": true + }, + { + "task_id": 202, + "ok": false + }, + { + "task_id": 203, + "ok": true + }, + { + "task_id": 204, + "ok": true + }, + { + "task_id": 205, + "ok": false + }, + { + "task_id": 206, + "ok": false + }, + { + "task_id": 207, + "ok": false + }, + { + "task_id": 208, + "ok": false + }, + { + "task_id": 209, + "ok": false + }, + { + "task_id": 210, + "ok": false + }, + { + "task_id": 211, + "ok": false + }, + { + "task_id": 212, + "ok": true + }, + { + "task_id": 213, + "ok": true + }, + { + "task_id": 214, + "ok": true + }, + { + "task_id": 215, + "ok": true + }, + { + "task_id": 216, + "ok": false + }, + { + "task_id": 217, + "ok": true + }, + { + "task_id": 218, + "ok": false + }, + { + "task_id": 219, + "ok": false + }, + { + "task_id": 220, + "ok": false + }, + { + "task_id": 221, + "ok": true + }, + { + "task_id": 222, + "ok": true + }, + { + "task_id": 223, + "ok": false + }, + { + "task_id": 224, + "ok": true + }, + { + "task_id": 225, + "ok": true + }, + { + "task_id": 226, + "ok": true + }, + { + "task_id": 227, + "ok": true + }, + { + "task_id": 228, + "ok": false + }, + { + "task_id": 229, + "ok": false + }, + { + "task_id": 230, + "ok": true + }, + { + "task_id": 231, + "ok": true + }, + { + "task_id": 232, + "ok": true + }, + { + "task_id": 233, + "ok": false + }, + { + "task_id": 234, + "ok": true + }, + { + "task_id": 235, + "ok": false + }, + { + "task_id": 236, + "ok": false + }, + { + "task_id": 237, + "ok": false + }, + { + "task_id": 238, + "ok": true + }, + { + "task_id": 239, + "ok": false + }, + { + "task_id": 240, + "ok": false + }, + { + "task_id": 241, + "ok": false + }, + { + "task_id": 242, + "ok": true + }, + { + "task_id": 243, + "ok": false + }, + { + "task_id": 244, + "ok": true + }, + { + "task_id": 245, + "ok": false + }, + { + "task_id": 246, + "ok": true + }, + { + "task_id": 247, + "ok": false + }, + { + "task_id": 248, + "ok": false + }, + { + "task_id": 249, + "ok": false + }, + { + "task_id": 250, + "ok": true + }, + { + "task_id": 251, + "ok": true + }, + { + "task_id": 252, + "ok": true + }, + { + "task_id": 253, + "ok": true + }, + { + "task_id": 254, + "ok": false + }, + { + "task_id": 255, + "ok": false + }, + { + "task_id": 256, + "ok": true + }, + { + "task_id": 257, + "ok": true + }, + { + "task_id": 258, + "ok": true + }, + { + "task_id": 259, + "ok": false + }, + { + "task_id": 260, + "ok": false + }, + { + "task_id": 261, + "ok": true + }, + { + "task_id": 262, + "ok": true + }, + { + "task_id": 263, + "ok": true + }, + { + "task_id": 264, + "ok": false + }, + { + "task_id": 265, + "ok": true + }, + { + "task_id": 266, + "ok": true + }, + { + "task_id": 267, + "ok": true + }, + { + "task_id": 268, + "ok": false + }, + { + "task_id": 269, + "ok": true + }, + { + "task_id": 270, + "ok": true + }, + { + "task_id": 271, + "ok": true + }, + { + "task_id": 272, + "ok": true + }, + { + "task_id": 273, + "ok": true + }, + { + "task_id": 274, + "ok": true + }, + { + "task_id": 275, + "ok": false + }, + { + "task_id": 276, + "ok": false + }, + { + "task_id": 277, + "ok": false + }, + { + "task_id": 278, + "ok": true + }, + { + "task_id": 279, + "ok": false + }, + { + "task_id": 280, + "ok": true + }, + { + "task_id": 281, + "ok": true + }, + { + "task_id": 282, + "ok": true + }, + { + "task_id": 283, + "ok": true + }, + { + "task_id": 284, + "ok": true + }, + { + "task_id": 285, + "ok": true + }, + { + "task_id": 286, + "ok": false + }, + { + "task_id": 287, + "ok": true + }, + { + "task_id": 288, + "ok": false + }, + { + "task_id": 289, + "ok": false + }, + { + "task_id": 290, + "ok": true + }, + { + "task_id": 291, + "ok": false + }, + { + "task_id": 292, + "ok": true + }, + { + "task_id": 293, + "ok": true + }, + { + "task_id": 294, + "ok": true + }, + { + "task_id": 295, + "ok": false + }, + { + "task_id": 296, + "ok": false + }, + { + "task_id": 297, + "ok": false + }, + { + "task_id": 298, + "ok": false + }, + { + "task_id": 299, + "ok": false + }, + { + "task_id": 300, + "ok": false + }, + { + "task_id": 301, + "ok": true + }, + { + "task_id": 302, + "ok": true + }, + { + "task_id": 303, + "ok": false + }, + { + "task_id": 304, + "ok": false + }, + { + "task_id": 305, + "ok": false + }, + { + "task_id": 306, + "ok": false + }, + { + "task_id": 307, + "ok": false + }, + { + "task_id": 308, + "ok": true + }, + { + "task_id": 309, + "ok": true + }, + { + "task_id": 310, + "ok": false + }, + { + "task_id": 311, + "ok": false + }, + { + "task_id": 312, + "ok": false + }, + { + "task_id": 313, + "ok": false + }, + { + "task_id": 314, + "ok": false + }, + { + "task_id": 315, + "ok": true + }, + { + "task_id": 316, + "ok": true + }, + { + "task_id": 317, + "ok": true + }, + { + "task_id": 318, + "ok": false + }, + { + "task_id": 319, + "ok": true + }, + { + "task_id": 320, + "ok": false + }, + { + "task_id": 321, + "ok": false + }, + { + "task_id": 322, + "ok": true + }, + { + "task_id": 323, + "ok": false + }, + { + "task_id": 324, + "ok": false + }, + { + "task_id": 325, + "ok": false + }, + { + "task_id": 326, + "ok": false + }, + { + "task_id": 327, + "ok": true + }, + { + "task_id": 328, + "ok": false + }, + { + "task_id": 329, + "ok": true + }, + { + "task_id": 330, + "ok": true + }, + { + "task_id": 331, + "ok": false + }, + { + "task_id": 332, + "ok": true + }, + { + "task_id": 333, + "ok": true + }, + { + "task_id": 334, + "ok": true + }, + { + "task_id": 335, + "ok": false + }, + { + "task_id": 336, + "ok": true + }, + { + "task_id": 337, + "ok": false + }, + { + "task_id": 338, + "ok": true + }, + { + "task_id": 339, + "ok": false + }, + { + "task_id": 340, + "ok": true + }, + { + "task_id": 341, + "ok": true + }, + { + "task_id": 342, + "ok": false + }, + { + "task_id": 343, + "ok": false + }, + { + "task_id": 344, + "ok": false + }, + { + "task_id": 345, + "ok": true + }, + { + "task_id": 346, + "ok": false + }, + { + "task_id": 347, + "ok": true + }, + { + "task_id": 348, + "ok": false + }, + { + "task_id": 349, + "ok": true + }, + { + "task_id": 350, + "ok": true + }, + { + "task_id": 351, + "ok": false + }, + { + "task_id": 352, + "ok": true + }, + { + "task_id": 353, + "ok": true + }, + { + "task_id": 354, + "ok": true + }, + { + "task_id": 355, + "ok": false + }, + { + "task_id": 356, + "ok": true + }, + { + "task_id": 357, + "ok": true + }, + { + "task_id": 358, + "ok": true + }, + { + "task_id": 359, + "ok": false + }, + { + "task_id": 360, + "ok": false + }, + { + "task_id": 361, + "ok": true + }, + { + "task_id": 362, + "ok": false + }, + { + "task_id": 363, + "ok": true + }, + { + "task_id": 364, + "ok": false + }, + { + "task_id": 365, + "ok": true + }, + { + "task_id": 366, + "ok": true + }, + { + "task_id": 367, + "ok": false + }, + { + "task_id": 368, + "ok": true + }, + { + "task_id": 369, + "ok": true + }, + { + "task_id": 370, + "ok": false + }, + { + "task_id": 371, + "ok": false + }, + { + "task_id": 372, + "ok": false + }, + { + "task_id": 373, + "ok": true + }, + { + "task_id": 374, + "ok": false + }, + { + "task_id": 375, + "ok": false + }, + { + "task_id": 376, + "ok": false + }, + { + "task_id": 377, + "ok": true + }, + { + "task_id": 378, + "ok": true + }, + { + "task_id": 379, + "ok": true + }, + { + "task_id": 380, + "ok": false + }, + { + "task_id": 381, + "ok": true + }, + { + "task_id": 382, + "ok": false + }, + { + "task_id": 383, + "ok": false + }, + { + "task_id": 384, + "ok": false + }, + { + "task_id": 385, + "ok": true + }, + { + "task_id": 386, + "ok": false + }, + { + "task_id": 387, + "ok": true + }, + { + "task_id": 388, + "ok": true + }, + { + "task_id": 389, + "ok": true + }, + { + "task_id": 390, + "ok": false + }, + { + "task_id": 391, + "ok": false + }, + { + "task_id": 392, + "ok": false + }, + { + "task_id": 393, + "ok": true + }, + { + "task_id": 394, + "ok": true + }, + { + "task_id": 395, + "ok": true + }, + { + "task_id": 396, + "ok": false + }, + { + "task_id": 397, + "ok": true + }, + { + "task_id": 398, + "ok": true + }, + { + "task_id": 399, + "ok": true + }, + { + "task_id": 400, + "ok": false + }, + { + "task_id": 401, + "ok": true + }, + { + "task_id": 402, + "ok": false + }, + { + "task_id": 403, + "ok": false + }, + { + "task_id": 404, + "ok": true + }, + { + "task_id": 405, + "ok": true + }, + { + "task_id": 406, + "ok": true + }, + { + "task_id": 407, + "ok": false + }, + { + "task_id": 408, + "ok": false + }, + { + "task_id": 409, + "ok": false + }, + { + "task_id": 410, + "ok": true + }, + { + "task_id": 411, + "ok": false + }, + { + "task_id": 412, + "ok": true + }, + { + "task_id": 413, + "ok": true + }, + { + "task_id": 414, + "ok": true + }, + { + "task_id": 415, + "ok": false + }, + { + "task_id": 416, + "ok": false + }, + { + "task_id": 417, + "ok": false + }, + { + "task_id": 418, + "ok": true + }, + { + "task_id": 419, + "ok": true + }, + { + "task_id": 420, + "ok": true + }, + { + "task_id": 421, + "ok": true + }, + { + "task_id": 422, + "ok": true + }, + { + "task_id": 423, + "ok": false + }, + { + "task_id": 424, + "ok": true + }, + { + "task_id": 425, + "ok": true + }, + { + "task_id": 426, + "ok": true + }, + { + "task_id": 427, + "ok": true + }, + { + "task_id": 428, + "ok": true + }, + { + "task_id": 429, + "ok": false + }, + { + "task_id": 430, + "ok": false + }, + { + "task_id": 431, + "ok": true + }, + { + "task_id": 432, + "ok": false + }, + { + "task_id": 433, + "ok": true + }, + { + "task_id": 434, + "ok": false + }, + { + "task_id": 435, + "ok": true + }, + { + "task_id": 436, + "ok": false + }, + { + "task_id": 437, + "ok": false + }, + { + "task_id": 438, + "ok": false + }, + { + "task_id": 439, + "ok": true + }, + { + "task_id": 440, + "ok": false + }, + { + "task_id": 441, + "ok": true + }, + { + "task_id": 442, + "ok": true + }, + { + "task_id": 443, + "ok": false + }, + { + "task_id": 444, + "ok": false + }, + { + "task_id": 445, + "ok": false + }, + { + "task_id": 446, + "ok": true + }, + { + "task_id": 447, + "ok": true + }, + { + "task_id": 448, + "ok": false + }, + { + "task_id": 449, + "ok": false + }, + { + "task_id": 450, + "ok": true + }, + { + "task_id": 451, + "ok": true + }, + { + "task_id": 452, + "ok": true + }, + { + "task_id": 453, + "ok": true + }, + { + "task_id": 454, + "ok": true + }, + { + "task_id": 455, + "ok": true + }, + { + "task_id": 456, + "ok": true + }, + { + "task_id": 457, + "ok": true + }, + { + "task_id": 458, + "ok": true + }, + { + "task_id": 459, + "ok": false + }, + { + "task_id": 460, + "ok": true + }, + { + "task_id": 461, + "ok": false + }, + { + "task_id": 462, + "ok": false + }, + { + "task_id": 463, + "ok": false + }, + { + "task_id": 464, + "ok": false + }, + { + "task_id": 465, + "ok": true + }, + { + "task_id": 466, + "ok": true + }, + { + "task_id": 467, + "ok": false + }, + { + "task_id": 468, + "ok": true + }, + { + "task_id": 469, + "ok": false + }, + { + "task_id": 470, + "ok": false + }, + { + "task_id": 471, + "ok": false + }, + { + "task_id": 472, + "ok": false + }, + { + "task_id": 473, + "ok": false + }, + { + "task_id": 474, + "ok": true + }, + { + "task_id": 475, + "ok": true + }, + { + "task_id": 476, + "ok": true + }, + { + "task_id": 477, + "ok": true + }, + { + "task_id": 478, + "ok": true + }, + { + "task_id": 479, + "ok": true + }, + { + "task_id": 480, + "ok": true + }, + { + "task_id": 481, + "ok": false + }, + { + "task_id": 482, + "ok": true + }, + { + "task_id": 483, + "ok": false + }, + { + "task_id": 484, + "ok": true + }, + { + "task_id": 485, + "ok": true + }, + { + "task_id": 486, + "ok": false + }, + { + "task_id": 487, + "ok": true + }, + { + "task_id": 488, + "ok": false + }, + { + "task_id": 489, + "ok": false + }, + { + "task_id": 490, + "ok": false + }, + { + "task_id": 491, + "ok": true + }, + { + "task_id": 492, + "ok": true + }, + { + "task_id": 493, + "ok": false + }, + { + "task_id": 494, + "ok": true + }, + { + "task_id": 495, + "ok": true + }, + { + "task_id": 496, + "ok": false + }, + { + "task_id": 497, + "ok": false + }, + { + "task_id": 498, + "ok": true + }, + { + "task_id": 499, + "ok": true + }, + { + "task_id": 500, + "ok": false + }, + { + "task_id": 501, + "ok": false + }, + { + "task_id": 502, + "ok": true + }, + { + "task_id": 503, + "ok": false + }, + { + "task_id": 504, + "ok": true + }, + { + "task_id": 505, + "ok": true + }, + { + "task_id": 506, + "ok": true + }, + { + "task_id": 507, + "ok": true + }, + { + "task_id": 508, + "ok": false + }, + { + "task_id": 509, + "ok": true + }, + { + "task_id": 510, + "ok": true + } + ] +} \ No newline at end of file diff --git a/results-loop/stats_final.json b/results-loop/stats_final.json index 6f2454d..5a4513c 100644 --- a/results-loop/stats_final.json +++ b/results-loop/stats_final.json @@ -370,6 +370,30 @@ "a_only": 15, "b_only": 7, "p": 0.13380050659179688 + }, + "budget-cot vs loop k=4, overall": { + "n": 500, + "a_only": 36, + "b_only": 37, + "p": 1.0 + }, + "budget-cot vs loop k=4, hard": { + "n": 55, + "a_only": 14, + "b_only": 11, + "p": 0.6900379657745361 + }, + "budget-cot vs distill k=1, overall": { + "n": 500, + "a_only": 29, + "b_only": 26, + "p": 0.7877061896700435 + }, + "budget-cot vs distill k=1, hard": { + "n": 55, + "a_only": 8, + "b_only": 6, + "p": 0.79052734375 } }, "best-of-3 ORACLE (any-pass)": {