gpuq data contract: declarative per-job in/out sync (pull inputs, push outputs every 120s + at exit), gpuq_sync.sh helper

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nils
2026-07-16 01:17:33 +02:00
co-authored by Claude Fable 5
parent 684e793eef
commit 550d5cbe56
3 changed files with 93 additions and 6 deletions
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
# Selective repo<->bucket sync, same path convention as the worker's
# data contract (repo-relative path, mirrored at jspace:jspace/<path>).
#
# gpuq_sync.sh pull results-loop "eval_code_gate*.json"
# gpuq_sync.sh push results "jbar_e4b.pt"
# gpuq_sync.sh pull results-lens "" # whole directory
#
# Use from any machine with the rclone remote configured (typically the
# Spark) to fetch what nodes pushed, or to stage inputs nodes will pull.
set -eu
DIRN=$1
DIR=$2
GLOB=${3:-}
REPO="${GPUQ_REPO:-$HOME/jspace}"
INC=()
[ -n "$GLOB" ] && INC=(--include "$GLOB")
case "$DIRN" in
pull) rclone copy "jspace:jspace/$DIR" "$REPO/$DIR" "${INC[@]}" -v ;;
push) rclone copy "$REPO/$DIR" "jspace:jspace/$DIR" "${INC[@]}" -v ;;
*) echo "usage: gpuq_sync.sh pull|push <repo-relative-dir> [glob]"; exit 1 ;;
esac