30 lines
1.2 KiB
Python
30 lines
1.2 KiB
Python
"""Paired McNemar: carry-cot arm vs feedforward control, both grid cells."""
|
|
import json
|
|
from math import comb
|
|
from pathlib import Path
|
|
|
|
OUT = Path(__file__).resolve().parent.parent / "results-loop"
|
|
A = json.load(open(OUT / "eval_gsm_carrycot_e400.json"))
|
|
B = json.load(open(OUT / "eval_gsm_carrycot_ff_e400.json"))
|
|
star = {it["idx"]: it["label"]
|
|
for it in json.load(open(OUT / "star_data.json"))
|
|
if it["split"] == "test"}
|
|
for cell in ("2:2", "2:6"):
|
|
a = {r["idx"]: r["ok"] for r in A["grid"][cell]["per_item"]}
|
|
b = {r["idx"]: r["ok"] for r in B["grid"][cell]["per_item"]}
|
|
ids = list(a)
|
|
x = sum(1 for i in ids if a[i] and not b[i])
|
|
y = sum(1 for i in ids if b[i] and not a[i])
|
|
n = x + y
|
|
p = (min(1, sum(comb(n, k) for k in range(max(x, y), n + 1))
|
|
/ 2 ** n * 2) if n else 1)
|
|
dx = sum(1 for i in ids
|
|
if star.get(i) == "drop" and a[i] and not b[i])
|
|
dy = sum(1 for i in ids
|
|
if star.get(i) == "drop" and b[i] and not a[i])
|
|
dn = dx + dy
|
|
dp = (min(1, sum(comb(dn, k) for k in range(max(dx, dy), dn + 1))
|
|
/ 2 ** dn * 2) if dn else 1)
|
|
print(f"{cell}: carry-only={x} ff-only={y} McNemar p={p:.4f} "
|
|
f"drop-bucket discordants {dx}-{dy} McNemar p={dp:.4f}")
|