From e5fc453005427380aff33edeee63e9948958bf6b Mon Sep 17 00:00:00 2001 From: Nils Date: Tue, 14 Jul 2026 17:13:35 +0200 Subject: [PATCH] pin gemma-4 model revisions (upstream 12B template regression: thought channel -> empty generations) Co-Authored-By: Claude Fable 5 --- LESSONS.md | 10 ++++++++++ scripts/node_setup.sh | 11 ++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/LESSONS.md b/LESSONS.md index e1b2724..10e3823 100644 --- a/LESSONS.md +++ b/LESSONS.md @@ -76,3 +76,13 @@ reproduction: [`RESULTS.md`](RESULTS.md). Everything on `google/gemma-4-E2B-it` 3. **Verify with the lens, gate with the labels**: the J-lens picks the band, measures whether loops compute, and diagnoses failures; STaR difficulty labels supervise both the curriculum and (next) the adaptive-depth gate. + +## 12. Pin model revisions on fresh nodes +Upstream updated google/gemma-4-12B-it mid-project: the new chat template +adds a `<|channel>thought` scaffold, and greedy generation closes the empty +thought channel and stops — every generation decodes to "". Symptom: +0/500 pass rates with rc=0 (looks like a harness bug, is a silent model +swap). E2B was unaffected. Fix: `hf download --revision ` + repoint +`refs/main` in the cache; node_setup.sh now pins both models (12B +0e2b1058…, E2B 9dbdf8a8…). Rule: any cross-node result assumes identical +model revisions — pin them, don't trust "main". diff --git a/scripts/node_setup.sh b/scripts/node_setup.sh index 65a0035..bf6f3e3 100755 --- a/scripts/node_setup.sh +++ b/scripts/node_setup.sh @@ -10,11 +10,16 @@ rclone copy jspace:jspace/results-loop /dev/shm/jspace/results-loop/ --include " export HF_HUB_ENABLE_HF_TRANSFER=1 # downloads can hang mid-transfer on some vast.ai networks (xet stall): # retry loop, resume is free -for m in google/gemma-4-12B-it google/gemma-4-E2B-it; do +# PIN revisions: upstream updated gemma-4-12B-it's chat template (added a +# thought channel) mid-project; unpinned downloads silently change behavior. +for spec in "google/gemma-4-12B-it 0e2b1058541244490925fbacf8972041435691ac" \ + "google/gemma-4-E2B-it 9dbdf8a839e4e9e0eb56ed80cc8886661d3817cf"; do + set -- $spec for i in 1 2 3 4 5; do - timeout 900 hf download "$m" && break - echo "RETRY $i: $m"; sleep 5 + timeout 900 hf download "$1" --revision "$2" && break + echo "RETRY $i: $1"; sleep 5 done + echo "$2" > "$HF_HOME/hub/models--${1//\//--}/refs/main" done # probe gate: one verified generation before anything batch export HF_HUB_OFFLINE=1 JLENS_MODEL=google/gemma-4-12B-it