Initial import of streamline
This commit is contained in:
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# executor-snapshot.sh — take a self-contained snapshot of a remote executor
|
||||
# (satellite) node so it can be restored on any other box with executor-restore.sh.
|
||||
#
|
||||
# Captured on the remote:
|
||||
# 1. Streamline repo (executor source + shared/ + bin/ + package.json + bun.lock)
|
||||
# Optionally node_modules (default: included) so restore doesn't need `bun install`.
|
||||
# Excludes .git/objects/pack (usually large) unless SNAPSHOT_KEEP_GIT=1.
|
||||
# Excludes runtime data (bug_workspace/ ai_workspace/ taskLog/ satellite/dist/)
|
||||
# unless SNAPSHOT_INCLUDE_WORKSPACES=1.
|
||||
# 2. Repo-level config: .env, .env.example, .mcp.json (if present), CLAUDE.md (if present).
|
||||
# 3. Agent CLI global configs & skills & MCP:
|
||||
# ~/.claude.json (global claude config incl. mcpServers, env, model)
|
||||
# ~/.claude/ (skills, settings.json, plugins/, sessions/… WITHOUT projects/, history.jsonl)
|
||||
# ~/.cursor/ (skills, skills-cursor, cli-config.json, agents/, hooks.json, mcp*)
|
||||
# ~/.trae/ (cli/*, plugins/, skills/, memories/)
|
||||
# ~/.agents/ (skills/, auth.json)
|
||||
# Large caches, sessions histories and per-project state that are not
|
||||
# needed to boot an executor are excluded — see EXCLUDE_HOME below.
|
||||
#
|
||||
# Output: a single local .tar.gz containing everything, plus a MANIFEST.txt
|
||||
# with the remote paths so the restore script knows where to put things.
|
||||
#
|
||||
# Auth: SSH with password, using sshpass. Install with:
|
||||
# brew install hudochenkov/sshpass/sshpass (macOS)
|
||||
# apt-get install sshpass (Debian/Ubuntu)
|
||||
#
|
||||
# Usage:
|
||||
# scripts/executor-snapshot.sh <user@host> <password> [--repo <remote_repo_path>] \
|
||||
# [--out <local_output.tar.gz>] \
|
||||
# [--no-node-modules] [--include-workspaces] [--port 22]
|
||||
#
|
||||
# Examples:
|
||||
# scripts/executor-snapshot.sh fei@fei-ios-01.shiro0.icu 'hunter2'
|
||||
# scripts/executor-snapshot.sh fei@10.0.0.5 'hunter2' --repo /home/fei/streamline --out ~/snapshots/ios01.tar.gz
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# ---------- argv ----------
|
||||
if [ $# -lt 2 ]; then
|
||||
sed -n '3,42p' "$0"
|
||||
exit 2
|
||||
fi
|
||||
SSH_TARGET="$1"; SSH_PASS="$2"; shift 2
|
||||
REMOTE_REPO="" # auto-detect if not provided
|
||||
OUT_PATH=""
|
||||
INCLUDE_NM=1
|
||||
INCLUDE_WORKSPACES=0
|
||||
SSH_PORT=22
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--repo) REMOTE_REPO="$2"; shift 2 ;;
|
||||
--out) OUT_PATH="$2"; shift 2 ;;
|
||||
--no-node-modules) INCLUDE_NM=0; shift ;;
|
||||
--include-node-modules) INCLUDE_NM=1; shift ;;
|
||||
--include-workspaces) INCLUDE_WORKSPACES=1; shift ;;
|
||||
--port) SSH_PORT="$2"; shift 2 ;;
|
||||
-h|--help) sed -n '3,42p' "$0"; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
REMOTE_HOST="${SSH_TARGET#*@}"
|
||||
if ! command -v sshpass >/dev/null 2>&1; then
|
||||
echo "sshpass not found — install it (brew install hudochenkov/sshpass/sshpass)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SSH_OPTS=(-p "$SSH_PORT" -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 -o ServerAliveInterval=30 -o LogLevel=ERROR)
|
||||
run_ssh() { sshpass -p "$SSH_PASS" ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "$@"; }
|
||||
run_scp() { sshpass -p "$SSH_PASS" scp -P "$SSH_PORT" -o StrictHostKeyChecking=accept-new -o LogLevel=ERROR "$@"; }
|
||||
|
||||
# ---------- discover remote repo ----------
|
||||
if [ -z "$REMOTE_REPO" ]; then
|
||||
echo "[snapshot] auto-detecting remote streamline repo…"
|
||||
REMOTE_REPO=$(run_ssh 'set -e
|
||||
# look for a running satellite process first
|
||||
pid=$(pgrep -f "satellite/src/index.ts" 2>/dev/null | head -n1 || true)
|
||||
if [ -n "$pid" ]; then
|
||||
# /proc/<pid>/cwd on linux, lsof fallback for macOS
|
||||
if [ -r "/proc/$pid/cwd" ]; then
|
||||
readlink -f "/proc/$pid/cwd" && exit 0
|
||||
fi
|
||||
cwd=$(lsof -p "$pid" 2>/dev/null | awk "\$4==\"cwd\"{print \$NF; exit}")
|
||||
[ -n "$cwd" ] && { echo "$cwd"; exit 0; }
|
||||
fi
|
||||
# fallbacks
|
||||
for cand in "$HOME/streamline" "$HOME/workspace/streamline" "/opt/streamline"; do
|
||||
[ -f "$cand/package.json" ] && [ -d "$cand/satellite" ] && { echo "$cand"; exit 0; }
|
||||
done
|
||||
exit 3
|
||||
') || {
|
||||
echo "[snapshot] could not auto-detect; pass --repo <path>" >&2
|
||||
exit 3
|
||||
}
|
||||
fi
|
||||
echo "[snapshot] remote repo: $REMOTE_REPO"
|
||||
|
||||
# ---------- default output path ----------
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
if [ -z "$OUT_PATH" ]; then
|
||||
OUT_DIR="$HOME/streamline-snapshots"
|
||||
mkdir -p "$OUT_DIR"
|
||||
OUT_PATH="$OUT_DIR/executor-${REMOTE_HOST//[^A-Za-z0-9]/_}-$STAMP.tar.gz"
|
||||
fi
|
||||
mkdir -p "$(dirname "$OUT_PATH")"
|
||||
|
||||
# ---------- exclusion patterns ----------
|
||||
REPO_EXCLUDES=(
|
||||
"--exclude=.git/objects/pack"
|
||||
"--exclude=satellite/dist"
|
||||
"--exclude=control-web/dist"
|
||||
"--exclude=control-web/node_modules/.cache"
|
||||
"--exclude=.DS_Store"
|
||||
"--exclude=log"
|
||||
"--exclude=logs"
|
||||
)
|
||||
[ "$INCLUDE_NM" -eq 0 ] && REPO_EXCLUDES+=("--exclude=node_modules")
|
||||
[ "$INCLUDE_WORKSPACES" -eq 0 ] && REPO_EXCLUDES+=("--exclude=bug_workspace" "--exclude=ai_workspace" "--exclude=taskLog")
|
||||
[ "${SNAPSHOT_KEEP_GIT:-0}" != "1" ] && REPO_EXCLUDES+=("--exclude=.git")
|
||||
|
||||
# What we want from $HOME/* (agent CLI configs, skills, MCP). Anything else
|
||||
# under those dirs (sessions, project caches, huge blob dumps) is excluded.
|
||||
HOME_INCLUDES=(
|
||||
".claude.json"
|
||||
".claude/settings.json"
|
||||
".claude/skills"
|
||||
".claude/plugins"
|
||||
".claude/CLAUDE.md"
|
||||
".cursor/cli-config.json"
|
||||
".cursor/skills"
|
||||
".cursor/skills-cursor"
|
||||
".cursor/agents"
|
||||
".cursor/hooks.json"
|
||||
".cursor/mcp.json"
|
||||
".cursor/argv.json"
|
||||
".trae/cli/traecli.toml"
|
||||
".trae/cli/traecli.yaml"
|
||||
".trae/cli/skill-config.json"
|
||||
".trae/cli/skills"
|
||||
".trae/cli/rules"
|
||||
".trae/skills"
|
||||
".trae/plugins"
|
||||
".agents/skills"
|
||||
".agents/auth.json"
|
||||
)
|
||||
|
||||
# ---------- build tar remotely, stream to local file ----------
|
||||
REMOTE_TMP=$(run_ssh 'mktemp -d "$HOME/.streamline-snap.XXXXXX"')
|
||||
echo "[snapshot] remote staging: $REMOTE_TMP"
|
||||
|
||||
# Write manifest on the remote, then tar the repo + selected $HOME entries and pipe here.
|
||||
run_ssh bash -s -- "$REMOTE_TMP" "$REMOTE_REPO" "$INCLUDE_NM" "$INCLUDE_WORKSPACES" <<'REMOTE_BOOTSTRAP'
|
||||
set -euo pipefail
|
||||
tmp="$1"; repo="$2"; inc_nm="$3"; inc_ws="$4"
|
||||
mkdir -p "$tmp"
|
||||
{
|
||||
echo "SCHEMA=1"
|
||||
echo "STAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "HOST=$(hostname)"
|
||||
echo "USER=$(id -un)"
|
||||
echo "REPO=$repo"
|
||||
echo "HOME=$HOME"
|
||||
echo "INCLUDE_NODE_MODULES=$inc_nm"
|
||||
echo "INCLUDE_WORKSPACES=$inc_ws"
|
||||
echo "SATELLITE_PIDS=$(pgrep -f 'satellite/src/index.ts' 2>/dev/null | tr '\n' ',' || true)"
|
||||
echo "BUN=$(command -v bun || true)"
|
||||
echo "CLAUDE=$(command -v claude || true)"
|
||||
echo "CURSOR=$(command -v cursor-agent || true)"
|
||||
echo "CODEX=$(command -v codex || true)"
|
||||
echo "UNAME=$(uname -a)"
|
||||
} > "$tmp/MANIFEST.txt"
|
||||
|
||||
# also record which HOME entries exist (so restore only touches captured paths)
|
||||
: > "$tmp/HOME_ENTRIES.txt"
|
||||
REMOTE_BOOTSTRAP
|
||||
|
||||
# Ship the HOME include list to the remote so it selects only what exists.
|
||||
printf '%s\n' "${HOME_INCLUDES[@]}" | \
|
||||
sshpass -p "$SSH_PASS" ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "cat > '$REMOTE_TMP/HOME_LIST.txt'"
|
||||
|
||||
echo "[snapshot] packing on remote and streaming to: $OUT_PATH"
|
||||
# Build a single tar with two top-level dirs: repo/ and home/, and a MANIFEST.txt.
|
||||
# We use `tar --transform` for GNU tar; on macOS `bsdtar` uses `-s` instead.
|
||||
# Detect once on the remote.
|
||||
run_ssh bash -s -- "$REMOTE_TMP" "$REMOTE_REPO" "${REPO_EXCLUDES[@]}" <<'REMOTE_TAR' > "$OUT_PATH"
|
||||
set -euo pipefail
|
||||
tmp="$1"; repo="$2"; shift 2
|
||||
excludes=("$@")
|
||||
|
||||
# Filter HOME entries to what actually exists; record for the manifest.
|
||||
# Portable across bash 3.2 (macOS) — no mapfile.
|
||||
present=()
|
||||
while IFS= read -r rel || [ -n "$rel" ]; do
|
||||
[ -z "$rel" ] && continue
|
||||
if [ -e "$HOME/$rel" ]; then present+=("$rel"); fi
|
||||
done < "$tmp/HOME_LIST.txt"
|
||||
: > "$tmp/HOME_ENTRIES.txt"
|
||||
for rel in "${present[@]:-}"; do
|
||||
[ -n "$rel" ] && echo "$rel" >> "$tmp/HOME_ENTRIES.txt"
|
||||
done
|
||||
|
||||
# Build a symlink tree under $tmp/snapshot/ that mirrors the archive layout.
|
||||
# Then tar the tree with -h (follow symlinks) — works with BSD and GNU tar
|
||||
# without any path-transform gymnastics.
|
||||
snap="$tmp/snapshot"
|
||||
rm -rf "$snap"
|
||||
mkdir -p "$snap/home"
|
||||
ln -s "$repo" "$snap/streamline"
|
||||
for rel in "${present[@]:-}"; do
|
||||
[ -z "$rel" ] && continue
|
||||
mkdir -p "$snap/home/$(dirname "$rel")"
|
||||
ln -s "$HOME/$rel" "$snap/home/$rel"
|
||||
done
|
||||
cp "$tmp/MANIFEST.txt" "$snap/MANIFEST.txt"
|
||||
cp "$tmp/HOME_ENTRIES.txt" "$snap/HOME_ENTRIES.txt"
|
||||
|
||||
# -h follows symlinks so contents are packed under snapshot/*.
|
||||
tar "${excludes[@]}" -czhf - -C "$tmp" snapshot 2>/dev/null
|
||||
REMOTE_TAR
|
||||
|
||||
# ---------- clean remote staging ----------
|
||||
run_ssh "rm -rf '$REMOTE_TMP'" || true
|
||||
|
||||
# ---------- verify locally ----------
|
||||
if ! [ -s "$OUT_PATH" ]; then
|
||||
echo "[snapshot] failed: output is empty ($OUT_PATH)" >&2
|
||||
exit 4
|
||||
fi
|
||||
size=$(du -h "$OUT_PATH" | awk '{print $1}')
|
||||
echo "[snapshot] archive: $OUT_PATH ($size)"
|
||||
|
||||
# Show the manifest for the user.
|
||||
tar -xzf "$OUT_PATH" -O snapshot/MANIFEST.txt 2>/dev/null | sed 's/^/ /'
|
||||
echo "[snapshot] done."
|
||||
Reference in New Issue
Block a user