gpuq data contract: declarative per-job in/out sync (pull inputs, push outputs every 120s + at exit), gpuq_sync.sh helper
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
# Selective repo<->bucket sync, same path convention as the worker's
|
||||
# data contract (repo-relative path, mirrored at jspace:jspace/<path>).
|
||||
#
|
||||
# gpuq_sync.sh pull results-loop "eval_code_gate*.json"
|
||||
# gpuq_sync.sh push results "jbar_e4b.pt"
|
||||
# gpuq_sync.sh pull results-lens "" # whole directory
|
||||
#
|
||||
# Use from any machine with the rclone remote configured (typically the
|
||||
# Spark) to fetch what nodes pushed, or to stage inputs nodes will pull.
|
||||
|
||||
set -eu
|
||||
DIRN=$1
|
||||
DIR=$2
|
||||
GLOB=${3:-}
|
||||
REPO="${GPUQ_REPO:-$HOME/jspace}"
|
||||
INC=()
|
||||
[ -n "$GLOB" ] && INC=(--include "$GLOB")
|
||||
case "$DIRN" in
|
||||
pull) rclone copy "jspace:jspace/$DIR" "$REPO/$DIR" "${INC[@]}" -v ;;
|
||||
push) rclone copy "$REPO/$DIR" "jspace:jspace/$DIR" "${INC[@]}" -v ;;
|
||||
*) echo "usage: gpuq_sync.sh pull|push <repo-relative-dir> [glob]"; exit 1 ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user