Files
streamline/scripts/executor-snapshot.sh
T
2026-11-16 17:26:44 -08:00

237 lines
8.8 KiB
Bash

#!/usr/bin/env bash
#
# executor-snapshot.sh — take a self-contained snapshot of a remote executor
# (satellite) node so it can be restored on any other box with executor-restore.sh.
#
# Captured on the remote:
# 1. Streamline repo (executor source + shared/ + bin/ + package.json + bun.lock)
# Optionally node_modules (default: included) so restore doesn't need `bun install`.
# Excludes .git/objects/pack (usually large) unless SNAPSHOT_KEEP_GIT=1.
# Excludes runtime data (bug_workspace/ ai_workspace/ taskLog/ satellite/dist/)
# unless SNAPSHOT_INCLUDE_WORKSPACES=1.
# 2. Repo-level config: .env, .env.example, .mcp.json (if present), CLAUDE.md (if present).
# 3. Agent CLI global configs & skills & MCP:
# ~/.claude.json (global claude config incl. mcpServers, env, model)
# ~/.claude/ (skills, settings.json, plugins/, sessions/… WITHOUT projects/, history.jsonl)
# ~/.cursor/ (skills, skills-cursor, cli-config.json, agents/, hooks.json, mcp*)
# ~/.trae/ (cli/*, plugins/, skills/, memories/)
# ~/.agents/ (skills/, auth.json)
# Large caches, sessions histories and per-project state that are not
# needed to boot an executor are excluded — see EXCLUDE_HOME below.
#
# Output: a single local .tar.gz containing everything, plus a MANIFEST.txt
# with the remote paths so the restore script knows where to put things.
#
# Auth: SSH with password, using sshpass. Install with:
# brew install hudochenkov/sshpass/sshpass (macOS)
# apt-get install sshpass (Debian/Ubuntu)
#
# Usage:
# scripts/executor-snapshot.sh <user@host> <password> [--repo <remote_repo_path>] \
# [--out <local_output.tar.gz>] \
# [--no-node-modules] [--include-workspaces] [--port 22]
#
# Examples:
# scripts/executor-snapshot.sh fei@fei-ios-01.shiro0.icu 'hunter2'
# scripts/executor-snapshot.sh fei@10.0.0.5 'hunter2' --repo /home/fei/streamline --out ~/snapshots/ios01.tar.gz
#
set -euo pipefail
# ---------- argv ----------
if [ $# -lt 2 ]; then
sed -n '3,42p' "$0"
exit 2
fi
SSH_TARGET="$1"; SSH_PASS="$2"; shift 2
REMOTE_REPO="" # auto-detect if not provided
OUT_PATH=""
INCLUDE_NM=1
INCLUDE_WORKSPACES=0
SSH_PORT=22
while [ $# -gt 0 ]; do
case "$1" in
--repo) REMOTE_REPO="$2"; shift 2 ;;
--out) OUT_PATH="$2"; shift 2 ;;
--no-node-modules) INCLUDE_NM=0; shift ;;
--include-node-modules) INCLUDE_NM=1; shift ;;
--include-workspaces) INCLUDE_WORKSPACES=1; shift ;;
--port) SSH_PORT="$2"; shift 2 ;;
-h|--help) sed -n '3,42p' "$0"; exit 0 ;;
*) echo "unknown arg: $1" >&2; exit 2 ;;
esac
done
REMOTE_HOST="${SSH_TARGET#*@}"
if ! command -v sshpass >/dev/null 2>&1; then
echo "sshpass not found — install it (brew install hudochenkov/sshpass/sshpass)" >&2
exit 1
fi
SSH_OPTS=(-p "$SSH_PORT" -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 -o ServerAliveInterval=30 -o LogLevel=ERROR)
run_ssh() { sshpass -p "$SSH_PASS" ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "$@"; }
run_scp() { sshpass -p "$SSH_PASS" scp -P "$SSH_PORT" -o StrictHostKeyChecking=accept-new -o LogLevel=ERROR "$@"; }
# ---------- discover remote repo ----------
if [ -z "$REMOTE_REPO" ]; then
echo "[snapshot] auto-detecting remote streamline repo…"
REMOTE_REPO=$(run_ssh 'set -e
# look for a running satellite process first
pid=$(pgrep -f "satellite/src/index.ts" 2>/dev/null | head -n1 || true)
if [ -n "$pid" ]; then
# /proc/<pid>/cwd on linux, lsof fallback for macOS
if [ -r "/proc/$pid/cwd" ]; then
readlink -f "/proc/$pid/cwd" && exit 0
fi
cwd=$(lsof -p "$pid" 2>/dev/null | awk "\$4==\"cwd\"{print \$NF; exit}")
[ -n "$cwd" ] && { echo "$cwd"; exit 0; }
fi
# fallbacks
for cand in "$HOME/streamline" "$HOME/workspace/streamline" "/opt/streamline"; do
[ -f "$cand/package.json" ] && [ -d "$cand/satellite" ] && { echo "$cand"; exit 0; }
done
exit 3
') || {
echo "[snapshot] could not auto-detect; pass --repo <path>" >&2
exit 3
}
fi
echo "[snapshot] remote repo: $REMOTE_REPO"
# ---------- default output path ----------
STAMP="$(date +%Y%m%d-%H%M%S)"
if [ -z "$OUT_PATH" ]; then
OUT_DIR="$HOME/streamline-snapshots"
mkdir -p "$OUT_DIR"
OUT_PATH="$OUT_DIR/executor-${REMOTE_HOST//[^A-Za-z0-9]/_}-$STAMP.tar.gz"
fi
mkdir -p "$(dirname "$OUT_PATH")"
# ---------- exclusion patterns ----------
REPO_EXCLUDES=(
"--exclude=.git/objects/pack"
"--exclude=satellite/dist"
"--exclude=control-web/dist"
"--exclude=control-web/node_modules/.cache"
"--exclude=.DS_Store"
"--exclude=log"
"--exclude=logs"
)
[ "$INCLUDE_NM" -eq 0 ] && REPO_EXCLUDES+=("--exclude=node_modules")
[ "$INCLUDE_WORKSPACES" -eq 0 ] && REPO_EXCLUDES+=("--exclude=bug_workspace" "--exclude=ai_workspace" "--exclude=taskLog")
[ "${SNAPSHOT_KEEP_GIT:-0}" != "1" ] && REPO_EXCLUDES+=("--exclude=.git")
# What we want from $HOME/* (agent CLI configs, skills, MCP). Anything else
# under those dirs (sessions, project caches, huge blob dumps) is excluded.
HOME_INCLUDES=(
".claude.json"
".claude/settings.json"
".claude/skills"
".claude/plugins"
".claude/CLAUDE.md"
".cursor/cli-config.json"
".cursor/skills"
".cursor/skills-cursor"
".cursor/agents"
".cursor/hooks.json"
".cursor/mcp.json"
".cursor/argv.json"
".trae/cli/traecli.toml"
".trae/cli/traecli.yaml"
".trae/cli/skill-config.json"
".trae/cli/skills"
".trae/cli/rules"
".trae/skills"
".trae/plugins"
".agents/skills"
".agents/auth.json"
)
# ---------- build tar remotely, stream to local file ----------
REMOTE_TMP=$(run_ssh 'mktemp -d "$HOME/.streamline-snap.XXXXXX"')
echo "[snapshot] remote staging: $REMOTE_TMP"
# Write manifest on the remote, then tar the repo + selected $HOME entries and pipe here.
run_ssh bash -s -- "$REMOTE_TMP" "$REMOTE_REPO" "$INCLUDE_NM" "$INCLUDE_WORKSPACES" <<'REMOTE_BOOTSTRAP'
set -euo pipefail
tmp="$1"; repo="$2"; inc_nm="$3"; inc_ws="$4"
mkdir -p "$tmp"
{
echo "SCHEMA=1"
echo "STAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "HOST=$(hostname)"
echo "USER=$(id -un)"
echo "REPO=$repo"
echo "HOME=$HOME"
echo "INCLUDE_NODE_MODULES=$inc_nm"
echo "INCLUDE_WORKSPACES=$inc_ws"
echo "SATELLITE_PIDS=$(pgrep -f 'satellite/src/index.ts' 2>/dev/null | tr '\n' ',' || true)"
echo "BUN=$(command -v bun || true)"
echo "CLAUDE=$(command -v claude || true)"
echo "CURSOR=$(command -v cursor-agent || true)"
echo "CODEX=$(command -v codex || true)"
echo "UNAME=$(uname -a)"
} > "$tmp/MANIFEST.txt"
# also record which HOME entries exist (so restore only touches captured paths)
: > "$tmp/HOME_ENTRIES.txt"
REMOTE_BOOTSTRAP
# Ship the HOME include list to the remote so it selects only what exists.
printf '%s\n' "${HOME_INCLUDES[@]}" | \
sshpass -p "$SSH_PASS" ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "cat > '$REMOTE_TMP/HOME_LIST.txt'"
echo "[snapshot] packing on remote and streaming to: $OUT_PATH"
# Build a single tar with two top-level dirs: repo/ and home/, and a MANIFEST.txt.
# We use `tar --transform` for GNU tar; on macOS `bsdtar` uses `-s` instead.
# Detect once on the remote.
run_ssh bash -s -- "$REMOTE_TMP" "$REMOTE_REPO" "${REPO_EXCLUDES[@]}" <<'REMOTE_TAR' > "$OUT_PATH"
set -euo pipefail
tmp="$1"; repo="$2"; shift 2
excludes=("$@")
# Filter HOME entries to what actually exists; record for the manifest.
# Portable across bash 3.2 (macOS) — no mapfile.
present=()
while IFS= read -r rel || [ -n "$rel" ]; do
[ -z "$rel" ] && continue
if [ -e "$HOME/$rel" ]; then present+=("$rel"); fi
done < "$tmp/HOME_LIST.txt"
: > "$tmp/HOME_ENTRIES.txt"
for rel in "${present[@]:-}"; do
[ -n "$rel" ] && echo "$rel" >> "$tmp/HOME_ENTRIES.txt"
done
# Build a symlink tree under $tmp/snapshot/ that mirrors the archive layout.
# Then tar the tree with -h (follow symlinks) — works with BSD and GNU tar
# without any path-transform gymnastics.
snap="$tmp/snapshot"
rm -rf "$snap"
mkdir -p "$snap/home"
ln -s "$repo" "$snap/streamline"
for rel in "${present[@]:-}"; do
[ -z "$rel" ] && continue
mkdir -p "$snap/home/$(dirname "$rel")"
ln -s "$HOME/$rel" "$snap/home/$rel"
done
cp "$tmp/MANIFEST.txt" "$snap/MANIFEST.txt"
cp "$tmp/HOME_ENTRIES.txt" "$snap/HOME_ENTRIES.txt"
# -h follows symlinks so contents are packed under snapshot/*.
tar "${excludes[@]}" -czhf - -C "$tmp" snapshot 2>/dev/null
REMOTE_TAR
# ---------- clean remote staging ----------
run_ssh "rm -rf '$REMOTE_TMP'" || true
# ---------- verify locally ----------
if ! [ -s "$OUT_PATH" ]; then
echo "[snapshot] failed: output is empty ($OUT_PATH)" >&2
exit 4
fi
size=$(du -h "$OUT_PATH" | awk '{print $1}')
echo "[snapshot] archive: $OUT_PATH ($size)"
# Show the manifest for the user.
tar -xzf "$OUT_PATH" -O snapshot/MANIFEST.txt 2>/dev/null | sed 's/^/ /'
echo "[snapshot] done."