From d2d372c909a76bf64971c6f7ce0782896998c524 Mon Sep 17 00:00:00 2001 From: Nils Date: Tue, 14 Jul 2026 12:34:57 +0200 Subject: [PATCH] feedforward flag for HumanEval eval Co-Authored-By: Claude Fable 5 --- scripts/eval_humaneval.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/eval_humaneval.py b/scripts/eval_humaneval.py index 50289cf..062a530 100644 --- a/scripts/eval_humaneval.py +++ b/scripts/eval_humaneval.py @@ -57,7 +57,8 @@ def run_he_tests(code, item, timeout=10): @torch.no_grad() -def loop_eval(looper, adapter, tok, items, k, batch=8, max_new=380): +def loop_eval(looper, adapter, tok, items, k, batch=8, max_new=380, + feedforward=False): codes = [] for i in range(0, len(items), batch): torch.cuda.empty_cache() @@ -67,7 +68,8 @@ def loop_eval(looper, adapter, tok, items, k, batch=8, max_new=380): add_special_tokens=False).to("cuda") gen = looper.generate_frozen_prompt(adapter, tok, enc["input_ids"], k, max_new_tokens=max_new, - attention_mask=enc["attention_mask"]) + attention_mask=enc["attention_mask"], + feedforward=feedforward) for j in range(len(chunk)): codes.append(extract_code( tok.decode(gen[j, enc["input_ids"].shape[1]:], @@ -83,6 +85,7 @@ def main(): ap.add_argument("--adapter", default=None) ap.add_argument("--tag", default="he") ap.add_argument("--ks", default="0,2,4") + ap.add_argument("--feedforward", action="store_true") args = ap.parse_args() ks = [int(x) for x in args.ks.split(",")] @@ -118,7 +121,8 @@ def main(): k0_ok = None for k in ks: t0 = time.time() - oks = loop_eval(looper, adapter, tok, items, k) + oks = loop_eval(looper, adapter, tok, items, k, + feedforward=args.feedforward) if k == 0: k0_ok = oks hard = [i for i, it in enumerate(items)