diff --git a/scripts/gpuq_submit.sh b/scripts/gpuq_submit.sh index c8dd044..72e9b06 100755 --- a/scripts/gpuq_submit.sh +++ b/scripts/gpuq_submit.sh @@ -1,12 +1,13 @@ #!/bin/bash -# Submit a job to a gpuq worker, or check queue status. +# Submit jobs to gpuq workers. # -# gpuq_submit.sh [more jobfiles...] +# gpuq_submit.sh [...] # pin to a worker +# gpuq_submit.sh --class [...] # pick idlest in class # gpuq_submit.sh --status # -# Ordering: jobs run oldest-first by NAME within a worker's queue; prefix -# files 00_..., 01_... to force order. Load balancing is the submitter's -# job — spread files across worker-ids (no work stealing, no races). +# Class rosters live in the bucket: gpuq/_classes/.txt (one worker id +# per line). Scheduling: prefer workers with 0 pending AND <15% GPU util +# (from the node heartbeat); tie-break by fewest pending, then random. set -eu if [ "$1" = "--status" ]; then @@ -16,6 +17,33 @@ if [ "$1" = "--status" ]; then rclone lsl -R jspace:jspace/gpuq --include "done/*.sh" 2>/dev/null | sort -k2,3 | tail -20 exit 0 fi + +pick_worker() { # $1 = class + local best="" best_score=999999 + for w in $(rclone cat "jspace:jspace/gpuq/_classes/$1.txt" 2>/dev/null); do + local node=${w%-gpu*} gpu=${w##*-gpu} + local pend util score + pend=$(rclone lsf "jspace:jspace/gpuq/$w/pending/" --include "*.sh" 2>/dev/null | wc -l) + util=$(rclone cat "jspace:jspace/gpuq/_health/$node.txt" 2>/dev/null \ + | grep -E "^$gpu, " | head -1 | awk -F', ' '{print $2}' | tr -dc 0-9) + util=${util:-100} + score=$((pend * 100 + util)) + if [ "$score" -lt "$best_score" ]; then best=$w; best_score=$score; fi + done + echo "$best" +} + +if [ "$1" = "--class" ]; then + CLASS=$2; shift 2 + for f in "$@"; do + W=$(pick_worker "$CLASS") + [ -z "$W" ] && { echo "no workers in class $CLASS"; exit 1; } + rclone copy "$f" "jspace:jspace/gpuq/$W/pending/" -q + echo "queued $(basename $f) -> $W (class $CLASS)" + done + exit 0 +fi + ID=$1; shift for f in "$@"; do rclone copy "$f" "jspace:jspace/gpuq/$ID/pending/" -q