#!/usr/bin/env bash # --------------------------------------------------------------------------- # download-task.sh — download one Streamline task's whole execution workspace # (reports, every round's logs, evidence incl. videos, TestSpec/EffectiveSpec, # Conditions.jsonl, images, manifests) as .zip. # # bash scripts/download-task.sh [--out DIR] [--direct] [--node HOST] [--fallback] # # The zip is produced ON THE SATELLITE by its workspace-archive endpoint, so # the content matches tarring the per-task execution dir directly — minus the # TikTok source checkout, built APK/AAB/IPA and artifacts/ (hook/XP modules + # post-run instrumentation: jacoco, tracking/AppLog, open-audit), excluded # server-side. # # Two ways to reach that endpoint: # default via the control plane proxy /api///workspace-archive # (requires a deployed control plane) # --direct straight to the home satellite's /sat/.../workspace-archive # (only the satellite needs the new code; the node base URL is # resolved from read-only /api/nodes, or pinned with --node; # set CLUSTER_SECRET if the cluster requires the shared header) # # Requires company intranet access to the read API # (STREAMLINE_API, default https://streamline-internal-api.bytedance.net). # Read-only: every request is a GET. # # --fallback also enables a client-side reconstruction (report + logs list + # evidence one by one + DB-side readable report) for nodes still running an # older satellite without the workspace-archive endpoint. It cannot include # the workspace's images/ or be byte-identical to a dir tar. # --------------------------------------------------------------------------- set -uo pipefail BASE="${STREAMLINE_API:-https://streamline-internal-api.bytedance.net}" ID="" OUT_DIR="." FALLBACK=0 DIRECT=0 NODE_PIN="" while [ $# -gt 0 ]; do case "$1" in --out) OUT_DIR="$2"; shift 2 ;; --fallback) FALLBACK=1; shift ;; --direct) DIRECT=1; shift ;; --node) NODE_PIN="$2"; shift 2 ;; -*) echo "unknown flag: $1" >&2; exit 2 ;; *) ID="$1"; shift ;; esac done [ -n "$ID" ] || { echo "usage: download-task.sh [--out DIR] [--fallback]" >&2; exit 2; } # Accept pasted panel URLs too — pull the uuid out. if [[ "$ID" =~ ([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}) ]]; then ID="${BASH_REMATCH[1]}" else echo "not a uuid: $ID" >&2; exit 2 fi mkdir -p "$OUT_DIR" OUT_ZIP="$OUT_DIR/$ID.zip" TMP_ZIP="$OUT_ZIP.tmp" trap 'rm -f "$TMP_ZIP"' EXIT # --- 1. Which track does this id belong to? -------------------------------- KIND="" for k in tasks features security verifications prototypes; do code=$(curl -sS -m 20 -o /dev/null -w '%{http_code}' "$BASE/api/$k/$ID" 2>/dev/null) if [ "$code" = "200" ]; then KIND="$k"; break; fi done if [ -z "$KIND" ]; then echo "id not found as task/feature/security/verification/prototype on $BASE" >&2 exit 1 fi echo "id=$ID kind=$KIND" # --- 2. Fast path: the satellite zips its real workspace dir --------------- # Workspace key differs for bugs: satellites key bug dirs by meegoId||id. META_URL="$BASE/api/$KIND/$ID" ARCHIVE_URL="$BASE/api/$KIND/$ID/workspace-archive" EXTRA_HEADER=() NODE_DESC="control-plane proxy" if [ "$DIRECT" = "1" ]; then META_BODY="$(curl -sS -m 20 "$META_URL" 2>/dev/null || true)" NODE_ID_RESOLVED="$(printf '%s' "$META_BODY" | jq -r '.executorNodeId // empty' 2>/dev/null)" [ -n "$NODE_PIN" ] && NODE_ID_RESOLVED="$NODE_PIN" if [ -z "$NODE_ID_RESOLVED" ]; then echo "task has no executorNodeId (pre-cluster legacy record) — direct mode needs a home node; try without --direct" >&2 exit 1 fi # --node accepts either a registry node id (resolved via /api/nodes) or a # direct base URL (http://host:port). if [[ "$NODE_ID_RESOLVED" == http://* || "$NODE_ID_RESOLVED" == https://* ]]; then NODE_BASE="${NODE_ID_RESOLVED%/}" else NODE_BASE="$(curl -sS -m 20 "$BASE/api/nodes" 2>/dev/null \ | jq -r --arg n "$NODE_ID_RESOLVED" '(.nodes // . // [])[] | select(.id==$n) | .baseUrl' 2>/dev/null | head -1)" if [ -z "$NODE_BASE" ]; then echo "node $NODE_ID_RESOLVED not found in /api/nodes (offline/pruned?)" >&2 exit 1 fi fi case "$KIND" in tasks) KEY="$(printf '%s' "$META_BODY" | jq -r '.meegoId // .id' 2>/dev/null)" ARCHIVE_URL="$NODE_BASE/sat/ws/$KEY/workspace-archive" ;; features) ARCHIVE_URL="$NODE_BASE/sat/features/$ID/workspace-archive" ;; security) ARCHIVE_URL="$NODE_BASE/sat/security/$ID/workspace-archive" ;; verifications) ARCHIVE_URL="$NODE_BASE/sat/verifications/$ID/workspace-archive" ;; prototypes) ARCHIVE_URL="$NODE_BASE/sat/prototypes/$ID/workspace-archive" ;; esac NODE_DESC="$NODE_ID_RESOLVED ($NODE_BASE)" [ -n "${CLUSTER_SECRET:-}" ] && EXTRA_HEADER=(-H "x-cluster-secret: $CLUSTER_SECRET") fi echo "archive endpoint: $NODE_DESC" code=$(curl -sS -m 600 ${EXTRA_HEADER[@]+"${EXTRA_HEADER[@]}"} -o "$TMP_ZIP" -w '%{http_code}' \ "$ARCHIVE_URL" 2>/dev/null || true) is_zip() { head -c2 "$1" 2>/dev/null | grep -q $'PK'; } if [ "$code" = "200" ] && is_zip "$TMP_ZIP"; then # The readable report + full task record live in the control-plane DB, not in # the satellite workspace — append them so the zip is self-contained for # review too. Best-effort; needs local zip/unzip. if command -v python3 >/dev/null 2>&1; then STAGE="$(mktemp -d)" META="$STAGE/meta.json"; HTML="$STAGE/readable-report.html" curl -sS -m 30 -o "$META" "$BASE/api/$KIND/$ID" 2>/dev/null || true curl -sS -m 60 "$BASE/api/$KIND/$ID/readable-report" 2>/dev/null \ | jq -r '.html // empty' > "$HTML" 2>/dev/null || true [ -s "$META" ] || META="" [ -s "$HTML" ] || HTML="" if [ -n "$META" ] || [ -n "$HTML" ]; then python3 - "$TMP_ZIP" "$META" "$HTML" <<'PY' import os, sys, zipfile zp, meta, html = sys.argv[1], sys.argv[2], sys.argv[3] extras = {} if meta: extras["meta.json"] = meta if html: extras["readable-report.html"] = html with zipfile.ZipFile(zp) as zin: names = zin.namelist() prefix = names[0].split("/", 1)[0] if names else "" existing = set(names) with zipfile.ZipFile(zp + ".new", "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zout: for item in zin.infolist(): zout.writestr(item, zin.read(item.filename)) for arc, src in extras.items(): dest = f"{prefix}/{arc}" if prefix else arc if dest not in existing: zout.write(src, dest) os.replace(zp + ".new", zp) PY fi rm -rf "$STAGE" fi mv "$TMP_ZIP" "$OUT_ZIP" trap - EXIT echo "saved $(du -h "$OUT_ZIP" | awk '{print $1}') -> $OUT_ZIP" exit 0 fi echo "workspace-archive unavailable (http=$code) — satellite likely on old code" [ "$FALLBACK" = "1" ] || { echo "re-run with --fallback for piecewise download, or deploy the new satellite" >&2; exit 1; } # --- 3. Fallback: reconstruct from the per-file endpoints ------------------ echo "fallback: downloading pieces …" STAGE="$(mktemp -d)" trap 'rm -rf "$STAGE"; rm -f "$TMP_ZIP"' EXIT ROOT="$STAGE/$ID" mkdir -p "$ROOT/logs" "$ROOT/evidence" fetch() { # curl -sS -m 300 -o "$2.tmp" "$1" 2>/dev/null || { rm -f "$2.tmp"; return 1; } # Proxied 404s come back as a small {"error":...} JSON body — treat as absent. if head -c 256 "$2.tmp" 2>/dev/null | grep -q '^{"error"'; then rm -f "$2.tmp"; return 1; fi [ -s "$2.tmp" ] && mv "$2.tmp" "$2" || { rm -f "$2.tmp"; return 1; } } case "$KIND" in tasks) REPORT_PATH="fix-report"; REPORT_FILE="FixReport.md" ;; features) REPORT_PATH="fix-report"; REPORT_FILE="ImplReport.md" ;; security) REPORT_PATH="security-report"; REPORT_FILE="EvidenceReport.md" ;; verifications) REPORT_PATH="report"; REPORT_FILE="Verify-Report.md" ;; prototypes) REPORT_PATH="report"; REPORT_FILE="ProtoReport.md" ;; esac fetch "$BASE/api/$KIND/$ID" "$ROOT/meta.json" fetch "$BASE/api/$KIND/$ID/$REPORT_PATH" "$ROOT/$REPORT_FILE" [ "$KIND" = "prototypes" ] && fetch "$BASE/api/$KIND/$ID/script" "$ROOT/prototype.js" [ "$KIND" = "verifications" ] && fetch "$BASE/api/$KIND/$ID/audit-report" "$ROOT/audit-report.json" curl -sS -m 60 "$BASE/api/$KIND/$ID/readable-report" 2>/dev/null \ | jq -r '.html // empty' > "$ROOT/readable-report.html" 2>/dev/null || true [ -s "$ROOT/readable-report.html" ] || rm -f "$ROOT/readable-report.html" fetch "$BASE/api/$KIND/$ID/asking/history" "$ROOT/asking.json" fetch "$BASE/api/$KIND/$ID/conditions" "$ROOT/conditions.json" LOGS=0 if fetch "$BASE/api/$KIND/$ID/logs" "$ROOT/logs.json"; then while IFS=$'\t' read -r name; do [ -n "$name" ] || continue fetch "$BASE/api/$KIND/$ID/logs/$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1],safe=""))' "$name")?lines=200000" "$ROOT/logs/$name" LOGS=$((LOGS + 1)) done < <(jq -r '.files[]?.name' "$ROOT/logs.json" 2>/dev/null) fi # Old nodes (pre round-split logs) only expose the latest session/eval file. [ -f "$ROOT/logs/session.log" ] || fetch "$BASE/api/$KIND/$ID/log-file?lines=200000" "$ROOT/logs/session.log" [ -f "$ROOT/logs/eval.log" ] || fetch "$BASE/api/$KIND/$ID/eval-log-file?lines=200000" "$ROOT/logs/eval.log" EVID=0 if fetch "$BASE/api/$KIND/$ID/evidence" "$ROOT/evidence.json"; then while IFS=$'\t' read -r name url; do [ -n "$name" ] || continue if [ -n "$url" ]; then fetch "$url" "$ROOT/evidence/$name" else fetch "$BASE/api/$KIND/$ID/evidence-file/$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1],safe=""))' "$name")" "$ROOT/evidence/$name"; fi [ -s "$ROOT/evidence/$name" ] && EVID=$((EVID + 1)) done < <(jq -r '.files[]? | [.name, (.url // "")] | @tsv' "$ROOT/evidence.json" 2>/dev/null) fi [ "$EVID" -gt 0 ] || rmdir "$ROOT/evidence" 2>/dev/null || true # Python zipfile instead of the zip CLI: the latter aborts ("Zip file # structure invalid") on evidence names containing unusual characters, which # record-evidence occasionally produces. python3 - "$STAGE" "$ID" "$TMP_ZIP" <<'PY' import os, sys, zipfile stage, root, out = sys.argv[1], sys.argv[2], sys.argv[3] with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as z: for base, _, files in os.walk(os.path.join(stage, root)): for f in files: full = os.path.join(base, f) z.write(full, os.path.relpath(full, stage)) PY [ -s "$TMP_ZIP" ] || { echo "packing failed; pieces left in $ROOT" >&2; exit 1; } mv "$TMP_ZIP" "$OUT_ZIP" trap 'rm -rf "$STAGE"' EXIT rm -rf "$STAGE" echo "saved fallback zip (logs=$LOGS evidence=$EVID) $(du -h "$OUT_ZIP" | awk '{print $1}') -> $OUT_ZIP"