whiteboard microscopy on hard(795)/drop(430) specimens: FF slips at the 3-term sum (155), carry re-expands and defers (165 ✓); boards show live continuation-plan vs premature digit-commit at identical prefixes

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nils
2026-07-16 19:48:58 +02:00
co-authored by Claude Fable 5
parent f143c67b3d
commit 46b5e1b802
9 changed files with 90 additions and 3 deletions
+11 -3
View File
@@ -27,6 +27,14 @@ OUT = Path(os.environ.get("LOOP_OUT",
Path(__file__).resolve().parent.parent / "results-loop"))
Q = "A shirt costs $80 and is on sale for 15% off. How much does it cost?"
K, P = 2, 6
TAG = "discount"
if "--idx" in sys.argv:
import json
_i = int(sys.argv[sys.argv.index("--idx") + 1])
_star = {it["idx"]: it for it in json.load(open(OUT / "star_data.json"))
if it["split"] == "test"}
Q = _star[_i]["question"]
TAG = f"gsm{_i}"
FF = "--ff" in sys.argv
ADAPTER = OUT / ("adapter_carrycot_ff_e400.pt" if FF else "adapter_carrycot_e400.pt")
@@ -45,7 +53,7 @@ mask = torch.ones_like(ids)
with torch.no_grad():
gen = generate_carry_c(looper, adapter, tok, ids, mask, K, P,
max_new_tokens=48, feedforward=FF)
max_new_tokens=72, feedforward=FF)
text = tok.decode(gen[0, ids.shape[1] + P:], skip_special_tokens=False)
print("GENERATED:", repr(text), flush=True)
@@ -111,5 +119,5 @@ for i, lab in enumerate(poslab):
torch.save({"question": Q, "k": K, "p": P, "generated": text,
"band": band, "poslab": poslab, "table": table,
"digit_probs": probs},
OUT / ("probe_discount_ff.pt" if FF else "probe_discount.pt"))
print("\nsaved", OUT / ("probe_discount_ff.pt" if FF else "probe_discount.pt"))
OUT / (f"probe_{TAG}_ff.pt" if FF else f"probe_{TAG}.pt"))
print("\nsaved", OUT / (f"probe_{TAG}_ff.pt" if FF else f"probe_{TAG}.pt"))