From 5efe5b790ae88210859c18668579130d2dd1fb76 Mon Sep 17 00:00:00 2001
From: Nils <u797+claude@posteo.de>
Date: Tue, 14 Jul 2026 11:48:03 +0200
Subject: [PATCH] gpuq: bucket-backed per-GPU job queue (worker loop +
 submit/status helper)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 scripts/gpuq_submit.sh | 23 +++++++++++++++++++++++
 scripts/gpuq_worker.sh | 41 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100755 scripts/gpuq_submit.sh
 create mode 100755 scripts/gpuq_worker.sh

diff --git a/scripts/gpuq_submit.sh b/scripts/gpuq_submit.sh
new file mode 100755
index 0000000..c8dd044
--- /dev/null
+++ b/scripts/gpuq_submit.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Submit a job to a gpuq worker, or check queue status.
+#
+#   gpuq_submit.sh <worker-id> <jobfile.sh> [more jobfiles...]
+#   gpuq_submit.sh --status
+#
+# Ordering: jobs run oldest-first by NAME within a worker's queue; prefix
+# files 00_..., 01_... to force order. Load balancing is the submitter's
+# job — spread files across worker-ids (no work stealing, no races).
+
+set -eu
+if [ "$1" = "--status" ]; then
+  echo "== pending =="
+  rclone lsf -R jspace:jspace/gpuq --include "pending/*.sh" 2>/dev/null
+  echo "== done (last 20) =="
+  rclone lsl -R jspace:jspace/gpuq --include "done/*.sh" 2>/dev/null | sort -k2,3 | tail -20
+  exit 0
+fi
+ID=$1; shift
+for f in "$@"; do
+  rclone copy "$f" "jspace:jspace/gpuq/$ID/pending/" -q
+  echo "queued $(basename $f) -> $ID"
+done
diff --git a/scripts/gpuq_worker.sh b/scripts/gpuq_worker.sh
new file mode 100755
index 0000000..827ee5b
--- /dev/null
+++ b/scripts/gpuq_worker.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# Per-GPU work queue worker, fed from the S3 bucket (rclone remote "jspace").
+#
+# Usage:  gpuq_worker.sh <worker-id> <gpu-index>
+# e.g.:   gpuq_worker.sh node1-gpu3 3
+# Run ONE per GPU, each in its own tmux window:
+#   tmux new-window -t ssh_tmux -n gpuq3 'bash ~/jspace/scripts/gpuq_worker.sh node1-gpu3 3'
+#
+# Queue layout (per worker, race-free by construction — no work stealing):
+#   jspace:jspace/gpuq/<worker-id>/pending/<job>.sh   submitted jobs
+#   jspace:jspace/gpuq/<worker-id>/done/<job>.sh|.log finished jobs + logs
+# Jobs run oldest-first by name — prefix with 00_, 01_, ... to order them.
+#
+# Job contract: plain bash, repo at ~/jspace, venv at ~/jspace/.venv (or
+# /venv/main on vast images — job script picks), CUDA_VISIBLE_DEVICES is set
+# by the worker. Job handles its own LOOP_OUT / result rclone if needed.
+
+set -u
+ID=$1
+GPU=$2
+Q="jspace:jspace/gpuq/$ID"
+LOCAL="$HOME/gpuq/$ID"
+mkdir -p "$LOCAL/pending"
+
+echo "[gpuq $ID] worker up on GPU $GPU, polling $Q/pending"
+while true; do
+  rclone move "$Q/pending" "$LOCAL/pending" --include "*.sh" -q 2>/dev/null
+  job=$(ls "$LOCAL"/pending/*.sh 2>/dev/null | sort | head -1)
+  if [ -z "$job" ]; then
+    sleep 20
+    continue
+  fi
+  name=$(basename "$job" .sh)
+  echo "[gpuq $ID] $(date +%F_%H:%M:%S) START $name"
+  CUDA_VISIBLE_DEVICES=$GPU bash "$job" > "$LOCAL/$name.log" 2>&1
+  rc=$?
+  echo "[gpuq $ID] $(date +%F_%H:%M:%S) DONE  $name rc=$rc"
+  echo "rc=$rc" >> "$LOCAL/$name.log"
+  rclone copy "$LOCAL/$name.log" "$Q/done/" -q
+  rclone moveto "$job" "$Q/done/$name.sh" -q
+done