From 550d5cbe567c77f86ff708d1cc13ba2c899ef2ec Mon Sep 17 00:00:00 2001 From: Nils Date: Thu, 16 Jul 2026 01:17:33 +0200 Subject: [PATCH] gpuq data contract: declarative per-job in/out sync (pull inputs, push outputs every 120s + at exit), gpuq_sync.sh helper Co-Authored-By: Claude Fable 5 --- scripts/GPUQ.md | 27 +++++++++++++++++++++++ scripts/gpuq_sync.sh | 23 ++++++++++++++++++++ scripts/gpuq_worker.sh | 49 ++++++++++++++++++++++++++++++++++++------ 3 files changed, 93 insertions(+), 6 deletions(-) create mode 100755 scripts/gpuq_sync.sh diff --git a/scripts/GPUQ.md b/scripts/GPUQ.md index b5cf65f..3affc33 100644 --- a/scripts/GPUQ.md +++ b/scripts/GPUQ.md @@ -25,6 +25,33 @@ happens in the filename (`00_`, `01_`, …). The bucket is both transport and durable state: a dead node's pending jobs stay visible and can be moved to another worker with one `rclone move`. +## Data contract (selective sync) + +Jobs declare what they need and what they produce as comments; the worker +does the rest — no node needs everything, and results survive node death: + +``` +# gpuq-in: results-loop/mbpp_data.json results-loop/adapter_code.pt +# gpuq-out: results-loop/eval_code_gate*.json results/jbar_31b.pt +``` + +Paths are repo-relative, mirrored at `jspace:jspace/`. +Inputs are pulled before the job starts. Outputs are pushed every +`$GPUQ_SYNC_EVERY` seconds (default 120) while the job runs and once at +exit — a dying node loses at most one interval. The in-flight log is +also mirrored to `gpuq//running/` each cycle. Globs allowed in +basenames; repeat lines for more paths; jobs with no declarations sync +nothing (backward compatible). Manual counterpart from any machine: + +``` +scripts/gpuq_sync.sh pull results-loop "eval_code_gate*.json" +scripts/gpuq_sync.sh push results-loop "adapter_code.pt" +``` + +Nodes need the rclone remote configured for any of this; keep a scoped +(write-limited) key for rented nodes. Without credentials the worker +still runs jobs — sync lines just no-op. + ## Bucket layout ``` diff --git a/scripts/gpuq_sync.sh b/scripts/gpuq_sync.sh new file mode 100755 index 0000000..f3f4b35 --- /dev/null +++ b/scripts/gpuq_sync.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Selective repo<->bucket sync, same path convention as the worker's +# data contract (repo-relative path, mirrored at jspace:jspace/). +# +# gpuq_sync.sh pull results-loop "eval_code_gate*.json" +# gpuq_sync.sh push results "jbar_e4b.pt" +# gpuq_sync.sh pull results-lens "" # whole directory +# +# Use from any machine with the rclone remote configured (typically the +# Spark) to fetch what nodes pushed, or to stage inputs nodes will pull. + +set -eu +DIRN=$1 +DIR=$2 +GLOB=${3:-} +REPO="${GPUQ_REPO:-$HOME/jspace}" +INC=() +[ -n "$GLOB" ] && INC=(--include "$GLOB") +case "$DIRN" in + pull) rclone copy "jspace:jspace/$DIR" "$REPO/$DIR" "${INC[@]}" -v ;; + push) rclone copy "$REPO/$DIR" "jspace:jspace/$DIR" "${INC[@]}" -v ;; + *) echo "usage: gpuq_sync.sh pull|push [glob]"; exit 1 ;; +esac diff --git a/scripts/gpuq_worker.sh b/scripts/gpuq_worker.sh index 5beba0a..a367fc7 100755 --- a/scripts/gpuq_worker.sh +++ b/scripts/gpuq_worker.sh @@ -11,18 +11,45 @@ # jspace:jspace/gpuq//done/.sh|.log finished jobs + logs # Jobs run oldest-first by name — prefix with 00_, 01_, ... to order them. # -# Job contract: plain bash, repo at ~/jspace, venv at ~/jspace/.venv (or -# /venv/main on vast images — job script picks), CUDA_VISIBLE_DEVICES is set -# by the worker. Job handles its own LOOP_OUT / result rclone if needed. +# Job contract: plain bash, repo at $GPUQ_REPO (default ~/jspace), +# CUDA_VISIBLE_DEVICES is set by the worker. +# +# DATA CONTRACT (selective sync — declare in job-file comments): +# # gpuq-in: results-loop/mbpp_data.json results-loop/adapter_code.pt +# # gpuq-out: results-loop/eval_code_gate*.json results/jbar_31b.pt +# Paths are repo-relative and mirror to jspace:jspace/ in the +# bucket. Inputs are pulled before the job starts; outputs are pushed +# every SYNC_EVERY seconds while the job runs AND once at exit — a dying +# node loses at most one sync interval. Globs allowed in the basename. +# Repeat the comment lines to declare more paths. No declaration = no sync +# (old jobs keep working; they can still rclone by hand). set -u ID=$1 GPU=$2 Q="jspace:jspace/gpuq/$ID" LOCAL="$HOME/gpuq/$ID" +REPO="${GPUQ_REPO:-$HOME/jspace}" +SYNC_EVERY="${GPUQ_SYNC_EVERY:-120}" mkdir -p "$LOCAL/pending" -echo "[gpuq $ID] worker up on GPU $GPU, polling $Q/pending" +sync_paths() { # $1 = direction (in|out), $2 = job file + local dir line p + while IFS= read -r line; do + for p in $line; do + local d b + d=$(dirname "$p"); b=$(basename "$p") + mkdir -p "$REPO/$d" + if [ "$1" = in ]; then + rclone copy "jspace:jspace/$d" "$REPO/$d" --include "$b" -q 2>/dev/null + else + rclone copy "$REPO/$d" "jspace:jspace/$d" --include "$b" -q 2>/dev/null + fi + done + done < <(sed -n "s/^# gpuq-$1:[[:space:]]*//p" "$2") +} + +echo "[gpuq $ID] worker up on GPU $GPU, polling $Q/pending (repo $REPO)" while true; do rclone move "$Q/pending" "$LOCAL/pending" --include "*.sh" -q 2>/dev/null job=$(ls "$LOCAL"/pending/*.sh 2>/dev/null | sort | head -1) @@ -32,10 +59,20 @@ while true; do fi name=$(basename "$job" .sh) echo "[gpuq $ID] $(date +%F_%H:%M:%S) START $name" - CUDA_VISIBLE_DEVICES=$GPU bash "$job" 2>&1 | tee "$LOCAL/$name.log" - rc=${PIPESTATUS[0]} + sync_paths in "$job" + CUDA_VISIBLE_DEVICES=$GPU bash "$job" > "$LOCAL/$name.log" 2>&1 & + pid=$! + while kill -0 $pid 2>/dev/null; do + sleep "$SYNC_EVERY" + sync_paths out "$job" + rclone copy "$LOCAL/$name.log" "$Q/running/" -q 2>/dev/null + done + wait $pid + rc=$? echo "[gpuq $ID] $(date +%F_%H:%M:%S) DONE $name rc=$rc" echo "rc=$rc" >> "$LOCAL/$name.log" + sync_paths out "$job" rclone copy "$LOCAL/$name.log" "$Q/done/" -q + rclone deletefile "$Q/running/$name.log" -q 2>/dev/null rclone moveto "$job" "$Q/done/$name.sh" -q done