node kit: alpha flag through train/eval, carry env+width fixes, bootstrap script
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -40,9 +40,13 @@ ap.add_argument("--seed", type=int, default=0)
|
||||
ap.add_argument("--pause", type=int, default=0,
|
||||
help="pause-token control: p inert tokens after prompt, "
|
||||
"feedforward adapter, no recurrence")
|
||||
ap.add_argument("--alpha", type=float, default=0.3,
|
||||
help="merge weight (2B-tuned default 0.3; try 0.1-0.15 at 12B)")
|
||||
ARGS = ap.parse_args()
|
||||
SEED = ARGS.seed
|
||||
SUFFIX = (f"_s{SEED}" if SEED else "") + (f"_p{ARGS.pause}" if ARGS.pause else "")
|
||||
SUFFIX = ((f"_s{SEED}" if SEED else "")
|
||||
+ (f"_p{ARGS.pause}" if ARGS.pause else "")
|
||||
+ (f"_a{ARGS.alpha}" if ARGS.alpha != 0.3 else ""))
|
||||
PAUSE_ID = 6 # <unused0>
|
||||
|
||||
|
||||
@@ -101,7 +105,7 @@ def main():
|
||||
p.requires_grad_(False)
|
||||
looper = BandLooper(model)
|
||||
d = model.config.get_text_config().hidden_size
|
||||
adapter = MergeAdapter(d=d).cuda()
|
||||
adapter = MergeAdapter(d=d, alpha=ARGS.alpha).cuda()
|
||||
opt = torch.optim.AdamW(adapter.parameters(), lr=LR, weight_decay=0.01)
|
||||
|
||||
train = [it for it in data if it["split"] == "train"
|
||||
|
||||
Reference in New Issue
Block a user