From 74f04d124fbc3c6174e0fb1a0ad6058c5f904def Mon Sep 17 00:00:00 2001 From: Nils Date: Thu, 16 Jul 2026 01:51:44 +0200 Subject: [PATCH] node_bootstrap.sh v2: one-command node setup encoding every paid-for lesson (workspace-only, snapshot pinning, credential-aware sync, contract workers, graceful stop) + gpuq_presign for credential-less nodes Co-Authored-By: Claude Fable 5 --- scripts/gpuq_presign.py | 40 +++++++++++++++++ scripts/node_bootstrap.sh | 95 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 scripts/gpuq_presign.py create mode 100644 scripts/node_bootstrap.sh diff --git a/scripts/gpuq_presign.py b/scripts/gpuq_presign.py new file mode 100644 index 0000000..dca46a8 --- /dev/null +++ b/scripts/gpuq_presign.py @@ -0,0 +1,40 @@ +"""Generate a presigned S3 PUT URL for a gpuq job's output tarball. + +Reads the rclone remote's credentials locally (never leaves this machine); +the URL is a write-once capability to a single object name — safe to embed +in job files that run on untrusted rented nodes. + +Usage: gpuq_presign.py [--hours 48] +Prints the URL on stdout. +""" + +import argparse +import configparser +from pathlib import Path + +import boto3 +from botocore.config import Config + +ap = argparse.ArgumentParser() +ap.add_argument("worker") +ap.add_argument("job") +ap.add_argument("--hours", type=int, default=48) +ap.add_argument("--bucket", default="jspace") +args = ap.parse_args() + +cfg = configparser.ConfigParser() +cfg.read(Path.home() / ".config/rclone/rclone.conf") +r = cfg["jspace"] +s3 = boto3.client( + "s3", endpoint_url=r["endpoint"], + aws_access_key_id=r["access_key_id"], + aws_secret_access_key=r["secret_access_key"], + config=Config(signature_version="s3v4"), +) +url = s3.generate_presigned_url( + "put_object", + Params={"Bucket": args.bucket, + "Key": f"gpuq/{args.worker}/out/{args.job}_out.tgz"}, + ExpiresIn=args.hours * 3600, +) +print(url) diff --git a/scripts/node_bootstrap.sh b/scripts/node_bootstrap.sh new file mode 100644 index 0000000..65b629c --- /dev/null +++ b/scripts/node_bootstrap.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# ONE-COMMAND bootstrap for rented GPU nodes (vast.ai and similar). +# +# bash node_bootstrap.sh [num-gpus] [--dry-run] +# +# Encodes every lesson this project paid for (LESSONS.md, memory notes): +# - storage: /workspace (survives instance stop) — NEVER /dev/shm (RAM, +# wiped on stop). Aborts if /workspace is missing rather than degrade. +# - repo: init/fetch/checkout pattern (safe when the dir already exists; +# plain `git clone` fails silently into non-empty dirs). +# - models: pinned by revision, loaded by SNAPSHOT PATH (never refs/main; +# no HF_HUB_OFFLINE — it breaks dataset streaming). Use pin_model(). +# - sync: if ~/.config/rclone/rclone.conf exists (owner places it — never +# copied by automation), workers sync results directly to S3 via the +# gpuq data contract (# gpuq-in/out job comments) and a 120s sidecar +# mirrors logs. Without credentials the node still works; use presigned +# PUT URLs (scripts/gpuq_presign.py, run at home) embedded in jobs. +# - workers: one gpuq worker per GPU, graceful STOP sentinel +# (touch ~/gpuq//STOP), never kill mid-job. +set -u +NODE_ID=${1:?usage: node_bootstrap.sh [num-gpus] [--dry-run]} +NGPU=${2:-1} +DRY=${3:-} + +say() { echo "[bootstrap $NODE_ID] $*"; } + +# --- storage --------------------------------------------------------- +if [ ! -d /workspace ]; then + say "FATAL: /workspace missing — rent instances with a disk allocation" + exit 1 +fi +export GPUQ_REPO=/workspace/jspace +export HF_HOME=/workspace/hf +say "storage: repo=$GPUQ_REPO hf=$HF_HOME ($(df -h /workspace | awk 'NR==2{print $4}') free)" + +# --- credentials / sync mode ---------------------------------------- +if [ -f "$HOME/.config/rclone/rclone.conf" ]; then + SYNC_MODE=rclone +else + SYNC_MODE=presign +fi +say "sync mode: $SYNC_MODE" + +[ "$DRY" = "--dry-run" ] && { say "dry-run OK"; exit 0; } + +# --- python + deps --------------------------------------------------- +source /venv/main/bin/activate 2>/dev/null || true +uv pip install -q "transformers==5.13.*" datasets accelerate + +# --- repo ------------------------------------------------------------ +mkdir -p "$GPUQ_REPO" && cd "$GPUQ_REPO" +git init -q 2>/dev/null +git remote add origin https://git.draic.info/nils/jspace.git 2>/dev/null +git fetch -q origin main && git checkout -q -f FETCH_HEAD || { say "FATAL: repo fetch failed"; exit 1; } +mkdir -p "$GPUQ_REPO/results" "$GPUQ_REPO/results-loop" + +# --- model pinning helper (source this file to use it in job scripts) - +pin_model() { # pin_model ; echoes snapshot path + local M=$1 REV=$2 + for i in 1 2 3 4 5; do + timeout 3600 hf download "$M" --revision "$REV" >&2 && break + echo "RETRY $i: $M" >&2; sleep 5 + done + mkdir -p "$HF_HOME/hub/models--${M//\//--}/refs" + echo "$REV" > "$HF_HOME/hub/models--${M//\//--}/refs/main" + echo "$HF_HOME/hub/models--${M//\//--}/snapshots/$REV" +} +export -f pin_model + +# --- log sidecar (rclone mode only) ---------------------------------- +if [ "$SYNC_MODE" = rclone ] && ! pgrep -f "[n]ode_logsync" > /dev/null; then + cat > /root/node_logsync.sh </dev/null + sleep 120 +done +EOF + chmod +x /root/node_logsync.sh + setsid nohup /root/node_logsync.sh > /dev/null 2>&1 < /dev/null & + say "log sidecar started" +fi + +# --- workers --------------------------------------------------------- +for g in $(seq 0 $((NGPU - 1))); do + W="$NODE_ID-gpu$g" + if tmux has-session -t "gpuq_$W" 2>/dev/null; then + say "worker $W already running" + else + tmux new-session -d -s "gpuq_$W" \ + "GPUQ_REPO=$GPUQ_REPO bash $GPUQ_REPO/scripts/gpuq_worker.sh $W $g" + say "worker $W started (queue: jspace:jspace/gpuq/$W/pending)" + fi +done +say "NODE-READY (submit: gpuq_submit.sh $NODE_ID-gpu0 job.sh; drain: touch ~/gpuq//STOP)"