From cd8e94dda5429cdc89c309598b0ac0410450d155 Mon Sep 17 00:00:00 2001 From: Nils Date: Tue, 14 Jul 2026 12:47:23 +0200 Subject: [PATCH] gpuq: idle-time class dispatcher (single-writer, race-free) + docs Co-Authored-By: Claude Fable 5 --- scripts/GPUQ.md | 23 ++++++++++++++++++----- scripts/gpuq_dispatcher.sh | 37 +++++++++++++++++++++++++++++++++++++ scripts/gpuq_submit.sh | 6 ++---- 3 files changed, 57 insertions(+), 9 deletions(-) create mode 100755 scripts/gpuq_dispatcher.sh diff --git a/scripts/GPUQ.md b/scripts/GPUQ.md index b8928e2..b5cf65f 100644 --- a/scripts/GPUQ.md +++ b/scripts/GPUQ.md @@ -6,9 +6,10 @@ S3 bucket (rclone remote `jspace`, Hetzner). Three shell scripts, no daemon, no database, no cross-node networking. ``` -scripts/gpuq_worker.sh one per GPU runs jobs from its own queue -scripts/gpuq_supervisor.sh one per node restarts workers, heartbeats -scripts/gpuq_submit.sh anywhere enqueue jobs / show status +scripts/gpuq_worker.sh one per GPU runs jobs from its own queue +scripts/gpuq_supervisor.sh one per node restarts workers, heartbeats +scripts/gpuq_submit.sh anywhere enqueue jobs / show status +scripts/gpuq_dispatcher.sh ONE, always-on assigns class jobs to idle GPUs ``` ## Design in one paragraph @@ -160,7 +161,11 @@ key after each rental burst (already flagged in vast-ai-notes.md). **Daily usage (from the Spark or anywhere with the `jspace` remote):** ```bash -bash scripts/gpuq_submit.sh node3-gpu4 myjob.sh # submit to a specific GPU +bash scripts/gpuq_submit.sh --class h100 myjob.sh # PREFERRED: class queue; + # dispatcher assigns to the + # next idle GPU in class +bash scripts/gpuq_submit.sh node3-gpu4 myjob.sh # pin to a specific GPU + # (ordered chains only) bash scripts/gpuq_submit.sh --status # pending + last 20 done rclone cat jspace:jspace/gpuq/_health/node3.txt # heartbeat; stale >2 min = trouble rclone cat jspace:jspace/gpuq/node3-gpu4/done/myjob.log | tail -30 @@ -181,7 +186,15 @@ rclone cat jspace:jspace/gpuq/node3-gpu4/done/myjob.log | tail -30 5. Ordering on one GPU = filename sort (`00_train.sh`, `01_eval.sh`). Cross-GPU deps: submit after the prerequisite shows in `--status`, or add an `until rclone lsf ; do sleep 60; done` preamble. -6. Draining a node: stop submitting to its queues, wait for `--status` to +6. **Class scheduling**: `--class` drops jobs into + `gpuq/_class-pending//`; the single dispatcher (runs on the Spark, + `gpuq_dispatcher.sh`) moves each job to a concrete worker the moment one + is idle (empty pending + GPU util <15% + heartbeat fresher than 3 min). + Rosters: `gpuq/_classes/.txt`, one worker id per line — edit with + `rclone`; adding/removing fleet nodes touches only these files. One + dispatcher only: it is the single writer that makes class queues + race-free. If it dies, class jobs simply wait; restart it anywhere. +7. Draining a node: stop submitting to its queues, wait for `--status` to clear, `pkill -f gpuq_`, final `rclone copy` of its results dirs, then destroy the instance. Pending jobs of a dead node survive in the bucket: `rclone move jspace:jspace/gpuq/nodeX-gpuN/pending jspace:jspace/gpuq/nodeY-gpuM/pending`. diff --git a/scripts/gpuq_dispatcher.sh b/scripts/gpuq_dispatcher.sh new file mode 100755 index 0000000..092d672 --- /dev/null +++ b/scripts/gpuq_dispatcher.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# gpuq dispatcher: assigns class-queued jobs to idle workers, continuously. +# Run exactly ONE instance, on an always-on machine (the Spark): +# tmux/setsid bash scripts/gpuq_dispatcher.sh +# +# Submit side drops jobs in jspace:jspace/gpuq/_class-pending// ; +# this loop moves each job to a concrete worker's pending/ the moment a +# worker in that class is idle (empty pending + GPU util <15% + fresh +# heartbeat). Single dispatcher == single writer == no claim races. +# Assignment is a bucket-side move, so a dispatcher crash loses nothing. + +set -u +POLL=${1:-30} +echo "[gpuq-dispatch] up, polling ${POLL}s" +while true; do + for cls in $(rclone lsf jspace:jspace/gpuq/_classes/ 2>/dev/null | sed 's/\.txt$//'); do + jobs=$(rclone lsf "jspace:jspace/gpuq/_class-pending/$cls/" --include "*.sh" 2>/dev/null | sort) + [ -z "$jobs" ] && continue + for w in $(rclone cat "jspace:jspace/gpuq/_classes/$cls.txt" 2>/dev/null); do + job=$(echo "$jobs" | head -1); [ -z "$job" ] && break + node=${w%-gpu*}; gpu=${w##*-gpu} + hb=$(rclone cat "jspace:jspace/gpuq/_health/$node.txt" 2>/dev/null) || continue + # heartbeat freshness: first line is UTC date_%H:%M:%SZ + hbts=$(echo "$hb" | head -1); hbage=$(( $(date -u +%s) - $(date -u -d "${hbts//_/ }" +%s 2>/dev/null || echo 0) )) + [ "$hbage" -gt 180 ] && continue + util=$(echo "$hb" | grep -E "^$gpu, " | head -1 | awk -F', ' '{print $2}' | tr -dc 0-9) + pend=$(rclone lsf "jspace:jspace/gpuq/$w/pending/" --include "*.sh" 2>/dev/null | wc -l) + if [ "$pend" -eq 0 ] && [ "${util:-100}" -lt 15 ]; then + rclone moveto "jspace:jspace/gpuq/_class-pending/$cls/$job" \ + "jspace:jspace/gpuq/$w/pending/$job" -q \ + && echo "[gpuq-dispatch] $(date +%H:%M:%S) $job -> $w" + jobs=$(echo "$jobs" | tail -n +2) + fi + done + done + sleep "$POLL" +done diff --git a/scripts/gpuq_submit.sh b/scripts/gpuq_submit.sh index 72e9b06..ba73439 100755 --- a/scripts/gpuq_submit.sh +++ b/scripts/gpuq_submit.sh @@ -36,10 +36,8 @@ pick_worker() { # $1 = class if [ "$1" = "--class" ]; then CLASS=$2; shift 2 for f in "$@"; do - W=$(pick_worker "$CLASS") - [ -z "$W" ] && { echo "no workers in class $CLASS"; exit 1; } - rclone copy "$f" "jspace:jspace/gpuq/$W/pending/" -q - echo "queued $(basename $f) -> $W (class $CLASS)" + rclone copy "$f" "jspace:jspace/gpuq/_class-pending/$CLASS/" -q + echo "queued $(basename $f) -> class $CLASS (dispatcher assigns on idle)" done exit 0 fi