jspace/scripts/train_carry.py

"""Stage B / design C: train the merge adapter for prompt-prefill + carry.

GSM8K only (the task the prompt-only loop failed on). Sequence:
[prompt] [p x <unused0> pauses] [gold answer]; k=2 prefill loops on the
prompt; carry scan through pauses + answer; CE on answer tokens.
Curriculum: easy p=2, hard p=6 (hard needs the longer latent chain).
"""

import argparse
import json
import math
import random
import sys
import time
from pathlib import Path

import torch
import torch.nn.functional as F

from carry_common import PAUSE_ID, carry_logits
from loop_common import BandLooper, MergeAdapter, chat_prompt, DIRECT_SUFFIX

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from jlens.core import load_model  # noqa: E402

OUT = Path(__file__).resolve().parent.parent / "results-loop"
STEPS = 600
BATCH = 4
LR = 1e-3
WARMUP = 20
K_PREFILL = 2
P_BY_LABEL = {"easy": 2, "hard": 6}

ap = argparse.ArgumentParser()
ap.add_argument("--feedforward", action="store_true",
                help="pause-token control: adapter(e,e), no carry")
ap.add_argument("--seed", type=int, default=0)
ARGS = ap.parse_args()
SEED = ARGS.seed
TAG = "pausectl" if ARGS.feedforward else "carry"


def lr_at(step):
    if step < WARMUP:
        return LR * (step + 1) / WARMUP
    t = (step - WARMUP) / max(1, STEPS - WARMUP)
    return 1e-4 + 0.5 * (LR - 1e-4) * (1 + math.cos(math.pi * t))


def build_batch(tok, items, p, device="cuda"):
    seqs, labs, plens = [], [], []
    for it in items:
        pr = tok(chat_prompt(tok, it["question"], DIRECT_SUFFIX),
                 add_special_tokens=False)["input_ids"]
        a = tok(it["gold"] + "<end_of_turn>",
                add_special_tokens=False)["input_ids"]
        seqs.append(pr + [PAUSE_ID] * p + a)
        labs.append([-100] * (len(pr) + p) + a)
        plens.append(len(pr))
    T = max(len(s) for s in seqs)
    pad = tok.pad_token_id or 0
    ids = torch.full((len(seqs), T), pad, dtype=torch.long)
    lab = torch.full((len(seqs), T), -100, dtype=torch.long)
    msk = torch.zeros((len(seqs), T), dtype=torch.long)
    for i, (s, l) in enumerate(zip(seqs, labs)):
        ids[i, : len(s)] = torch.tensor(s)
        lab[i, : len(s)] = torch.tensor(l)
        msk[i, : len(s)] = 1
    return (ids.to(device), msk.to(device), lab.to(device),
            torch.tensor(plens, device=device))


@torch.no_grad()
def val_loss(looper, adapter, tok, items, p, k=K_PREFILL):
    tot, n = 0.0, 0
    for i in range(0, len(items), BATCH):
        ids, msk, lab, plens = build_batch(tok, items[i : i + BATCH], p)
        logits = carry_logits(looper, adapter, ids, msk, plens, k, feedforward=ARGS.feedforward)
        loss = F.cross_entropy(logits[:, :-1].flatten(0, 1).float(),
                               lab[:, 1:].flatten(), ignore_index=-100)
        tot += loss.item() * len(ids)
        n += len(ids)
    return tot / n


def main():
    rng = random.Random(SEED)
    torch.manual_seed(SEED)
    data = [it for it in json.load(open(OUT / "star_data.json"))
            if it["split"] == "train" and it["label"] != "drop"]

    model, tok = load_model(dtype=torch.bfloat16)
    for pp in model.parameters():
        pp.requires_grad_(False)
    looper = BandLooper(model)
    adapter = MergeAdapter().cuda()
    opt = torch.optim.AdamW(adapter.parameters(), lr=LR, weight_decay=0.01)

    keep = [it for it in data
            if len(tok(it["question"])["input_ids"]) + 30 <= 400]
    rng.shuffle(keep)
    pool = {l: [it for it in keep if it["label"] == l]
            for l in ("easy", "hard")}
    val = {l: pool[l][:16] for l in pool}
    pool = {l: pool[l][16:] for l in pool}
    print(f"pool: easy={len(pool['easy'])} hard={len(pool['hard'])}",
          flush=True)

    log = []
    t0 = time.time()
    for step in range(STEPS):
        lbl = ("easy", "hard")[step % 2]
        batch = rng.sample(pool[lbl], BATCH)
        p = P_BY_LABEL[lbl]
        ids, msk, lab, plens = build_batch(tok, batch, p)

        for g in opt.param_groups:
            g["lr"] = lr_at(step)
        logits = carry_logits(looper, adapter, ids, msk, plens, K_PREFILL, feedforward=ARGS.feedforward,
                              use_checkpoint=True)
        loss = F.cross_entropy(logits[:, :-1].flatten(0, 1).float(),
                               lab[:, 1:].flatten(), ignore_index=-100)
        opt.zero_grad(set_to_none=True)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(adapter.parameters(), 1.0)
        opt.step()

        log.append({"step": step, "label": lbl, "p": p, "loss": loss.item()})
        if step % 10 == 0:
            print(f"step {step:4d}  {lbl} p={p}  loss={loss.item():.4f}  "
                  f"({(time.time()-t0)/(step+1):.1f}s/step)", flush=True)
        if step % 100 == 99 or step == STEPS - 1:
            vals = {}
            for l in ("easy", "hard"):
                for pp_ in (0, 2, 6):
                    vals[f"{l}_p{pp_}"] = val_loss(looper, adapter, tok,
                                                   val[l], pp_)
            print(f"  val@{step}: " + "  ".join(
                f"{n}={v:.3f}" for n, v in sorted(vals.items())), flush=True)
            log.append({"step": step, "val": vals})
            torch.save(adapter.state_dict(),
                       OUT / f"adapter_{TAG}_e{step+1}.pt")
            json.dump(log, open(OUT / f"train_{TAG}_log.json", "w"), indent=1)

    print("done", flush=True)


if __name__ == "__main__":
    main()