From 378bb36b8c4419f0fc39f9db75fbb56ab691bad8 Mon Sep 17 00:00:00 2001 From: Nils Date: Tue, 14 Jul 2026 16:56:17 +0200 Subject: [PATCH] node_setup: retry+timeout around model downloads (xet mid-transfer stalls) Co-Authored-By: Claude Fable 5 --- scripts/node_setup.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/node_setup.sh b/scripts/node_setup.sh index fa1214d..65a0035 100755 --- a/scripts/node_setup.sh +++ b/scripts/node_setup.sh @@ -8,8 +8,14 @@ cd /dev/shm && git clone https://git.draic.info/nils/jspace.git 2>/dev/null || ( cd /dev/shm/jspace rclone copy jspace:jspace/results-loop /dev/shm/jspace/results-loop/ --include "*.json" --include "adapter_code_e399.pt" export HF_HUB_ENABLE_HF_TRANSFER=1 -hf download google/gemma-4-12B-it -hf download google/gemma-4-E2B-it +# downloads can hang mid-transfer on some vast.ai networks (xet stall): +# retry loop, resume is free +for m in google/gemma-4-12B-it google/gemma-4-E2B-it; do + for i in 1 2 3 4 5; do + timeout 900 hf download "$m" && break + echo "RETRY $i: $m"; sleep 5 + done +done # probe gate: one verified generation before anything batch export HF_HUB_OFFLINE=1 JLENS_MODEL=google/gemma-4-12B-it python - <<'PY'