diff --git a/scripts/gpuq_worker.sh b/scripts/gpuq_worker.sh index a367fc7..4f1ac33 100755 --- a/scripts/gpuq_worker.sh +++ b/scripts/gpuq_worker.sh @@ -23,6 +23,10 @@ # node loses at most one sync interval. Globs allowed in the basename. # Repeat the comment lines to declare more paths. No declaration = no sync # (old jobs keep working; they can still rclone by hand). +# +# GRACEFUL RESTART: touch $HOME/gpuq//STOP — the worker exits +# between jobs, never mid-job. NEVER kill the tmux session while a job +# runs (that killed a 3.5h scan once). set -u ID=$1 @@ -51,6 +55,11 @@ sync_paths() { # $1 = direction (in|out), $2 = job file echo "[gpuq $ID] worker up on GPU $GPU, polling $Q/pending (repo $REPO)" while true; do + if [ -e "$LOCAL/STOP" ]; then + rm -f "$LOCAL/STOP" + echo "[gpuq $ID] $(date +%F_%H:%M:%S) STOP sentinel — draining (no job running)" + break + fi rclone move "$Q/pending" "$LOCAL/pending" --include "*.sh" -q 2>/dev/null job=$(ls "$LOCAL"/pending/*.sh 2>/dev/null | sort | head -1) if [ -z "$job" ]; then