241 lines
10 KiB
Bash
241 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
# ---------------------------------------------------------------------------
|
|
# download-task.sh — download one Streamline task's whole execution workspace
|
|
# (reports, every round's logs, evidence incl. videos, TestSpec/EffectiveSpec,
|
|
# Conditions.jsonl, images, manifests) as <id>.zip.
|
|
#
|
|
# bash scripts/download-task.sh <id> [--out DIR] [--direct] [--node HOST] [--fallback]
|
|
#
|
|
# The zip is produced ON THE SATELLITE by its workspace-archive endpoint, so
|
|
# the content matches tarring the per-task execution dir directly — minus the
|
|
# TikTok source checkout, built APK/AAB/IPA and artifacts/ (hook/XP modules +
|
|
# post-run instrumentation: jacoco, tracking/AppLog, open-audit), excluded
|
|
# server-side.
|
|
#
|
|
# Two ways to reach that endpoint:
|
|
# default via the control plane proxy /api/<kind>/<id>/workspace-archive
|
|
# (requires a deployed control plane)
|
|
# --direct straight to the home satellite's /sat/.../workspace-archive
|
|
# (only the satellite needs the new code; the node base URL is
|
|
# resolved from read-only /api/nodes, or pinned with --node;
|
|
# set CLUSTER_SECRET if the cluster requires the shared header)
|
|
#
|
|
# Requires company intranet access to the read API
|
|
# (STREAMLINE_API, default https://streamline-internal-api.bytedance.net).
|
|
# Read-only: every request is a GET.
|
|
#
|
|
# --fallback also enables a client-side reconstruction (report + logs list +
|
|
# evidence one by one + DB-side readable report) for nodes still running an
|
|
# older satellite without the workspace-archive endpoint. It cannot include
|
|
# the workspace's images/ or be byte-identical to a dir tar.
|
|
# ---------------------------------------------------------------------------
|
|
set -uo pipefail
|
|
|
|
BASE="${STREAMLINE_API:-https://streamline-internal-api.bytedance.net}"
|
|
ID=""
|
|
OUT_DIR="."
|
|
FALLBACK=0
|
|
DIRECT=0
|
|
NODE_PIN=""
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--out) OUT_DIR="$2"; shift 2 ;;
|
|
--fallback) FALLBACK=1; shift ;;
|
|
--direct) DIRECT=1; shift ;;
|
|
--node) NODE_PIN="$2"; shift 2 ;;
|
|
-*) echo "unknown flag: $1" >&2; exit 2 ;;
|
|
*) ID="$1"; shift ;;
|
|
esac
|
|
done
|
|
|
|
[ -n "$ID" ] || { echo "usage: download-task.sh <task-id> [--out DIR] [--fallback]" >&2; exit 2; }
|
|
|
|
# Accept pasted panel URLs too — pull the uuid out.
|
|
if [[ "$ID" =~ ([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}) ]]; then
|
|
ID="${BASH_REMATCH[1]}"
|
|
else
|
|
echo "not a uuid: $ID" >&2; exit 2
|
|
fi
|
|
|
|
mkdir -p "$OUT_DIR"
|
|
OUT_ZIP="$OUT_DIR/$ID.zip"
|
|
TMP_ZIP="$OUT_ZIP.tmp"
|
|
trap 'rm -f "$TMP_ZIP"' EXIT
|
|
|
|
# --- 1. Which track does this id belong to? --------------------------------
|
|
KIND=""
|
|
for k in tasks features security verifications prototypes; do
|
|
code=$(curl -sS -m 20 -o /dev/null -w '%{http_code}' "$BASE/api/$k/$ID" 2>/dev/null)
|
|
if [ "$code" = "200" ]; then KIND="$k"; break; fi
|
|
done
|
|
|
|
if [ -z "$KIND" ]; then
|
|
echo "id not found as task/feature/security/verification/prototype on $BASE" >&2
|
|
exit 1
|
|
fi
|
|
echo "id=$ID kind=$KIND"
|
|
|
|
# --- 2. Fast path: the satellite zips its real workspace dir ---------------
|
|
# Workspace key differs for bugs: satellites key bug dirs by meegoId||id.
|
|
META_URL="$BASE/api/$KIND/$ID"
|
|
ARCHIVE_URL="$BASE/api/$KIND/$ID/workspace-archive"
|
|
EXTRA_HEADER=()
|
|
NODE_DESC="control-plane proxy"
|
|
|
|
if [ "$DIRECT" = "1" ]; then
|
|
META_BODY="$(curl -sS -m 20 "$META_URL" 2>/dev/null || true)"
|
|
NODE_ID_RESOLVED="$(printf '%s' "$META_BODY" | jq -r '.executorNodeId // empty' 2>/dev/null)"
|
|
[ -n "$NODE_PIN" ] && NODE_ID_RESOLVED="$NODE_PIN"
|
|
if [ -z "$NODE_ID_RESOLVED" ]; then
|
|
echo "task has no executorNodeId (pre-cluster legacy record) — direct mode needs a home node; try without --direct" >&2
|
|
exit 1
|
|
fi
|
|
# --node accepts either a registry node id (resolved via /api/nodes) or a
|
|
# direct base URL (http://host:port).
|
|
if [[ "$NODE_ID_RESOLVED" == http://* || "$NODE_ID_RESOLVED" == https://* ]]; then
|
|
NODE_BASE="${NODE_ID_RESOLVED%/}"
|
|
else
|
|
NODE_BASE="$(curl -sS -m 20 "$BASE/api/nodes" 2>/dev/null \
|
|
| jq -r --arg n "$NODE_ID_RESOLVED" '(.nodes // . // [])[] | select(.id==$n) | .baseUrl' 2>/dev/null | head -1)"
|
|
if [ -z "$NODE_BASE" ]; then
|
|
echo "node $NODE_ID_RESOLVED not found in /api/nodes (offline/pruned?)" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
case "$KIND" in
|
|
tasks)
|
|
KEY="$(printf '%s' "$META_BODY" | jq -r '.meegoId // .id' 2>/dev/null)"
|
|
ARCHIVE_URL="$NODE_BASE/sat/ws/$KEY/workspace-archive" ;;
|
|
features) ARCHIVE_URL="$NODE_BASE/sat/features/$ID/workspace-archive" ;;
|
|
security) ARCHIVE_URL="$NODE_BASE/sat/security/$ID/workspace-archive" ;;
|
|
verifications) ARCHIVE_URL="$NODE_BASE/sat/verifications/$ID/workspace-archive" ;;
|
|
prototypes) ARCHIVE_URL="$NODE_BASE/sat/prototypes/$ID/workspace-archive" ;;
|
|
esac
|
|
NODE_DESC="$NODE_ID_RESOLVED ($NODE_BASE)"
|
|
[ -n "${CLUSTER_SECRET:-}" ] && EXTRA_HEADER=(-H "x-cluster-secret: $CLUSTER_SECRET")
|
|
fi
|
|
|
|
echo "archive endpoint: $NODE_DESC"
|
|
code=$(curl -sS -m 600 ${EXTRA_HEADER[@]+"${EXTRA_HEADER[@]}"} -o "$TMP_ZIP" -w '%{http_code}' \
|
|
"$ARCHIVE_URL" 2>/dev/null || true)
|
|
|
|
is_zip() { head -c2 "$1" 2>/dev/null | grep -q $'PK'; }
|
|
|
|
if [ "$code" = "200" ] && is_zip "$TMP_ZIP"; then
|
|
# The readable report + full task record live in the control-plane DB, not in
|
|
# the satellite workspace — append them so the zip is self-contained for
|
|
# review too. Best-effort; needs local zip/unzip.
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
STAGE="$(mktemp -d)"
|
|
META="$STAGE/meta.json"; HTML="$STAGE/readable-report.html"
|
|
curl -sS -m 30 -o "$META" "$BASE/api/$KIND/$ID" 2>/dev/null || true
|
|
curl -sS -m 60 "$BASE/api/$KIND/$ID/readable-report" 2>/dev/null \
|
|
| jq -r '.html // empty' > "$HTML" 2>/dev/null || true
|
|
[ -s "$META" ] || META=""
|
|
[ -s "$HTML" ] || HTML=""
|
|
if [ -n "$META" ] || [ -n "$HTML" ]; then
|
|
python3 - "$TMP_ZIP" "$META" "$HTML" <<'PY'
|
|
import os, sys, zipfile
|
|
zp, meta, html = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
extras = {}
|
|
if meta: extras["meta.json"] = meta
|
|
if html: extras["readable-report.html"] = html
|
|
with zipfile.ZipFile(zp) as zin:
|
|
names = zin.namelist()
|
|
prefix = names[0].split("/", 1)[0] if names else ""
|
|
existing = set(names)
|
|
with zipfile.ZipFile(zp + ".new", "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zout:
|
|
for item in zin.infolist():
|
|
zout.writestr(item, zin.read(item.filename))
|
|
for arc, src in extras.items():
|
|
dest = f"{prefix}/{arc}" if prefix else arc
|
|
if dest not in existing:
|
|
zout.write(src, dest)
|
|
os.replace(zp + ".new", zp)
|
|
PY
|
|
fi
|
|
rm -rf "$STAGE"
|
|
fi
|
|
mv "$TMP_ZIP" "$OUT_ZIP"
|
|
trap - EXIT
|
|
echo "saved $(du -h "$OUT_ZIP" | awk '{print $1}') -> $OUT_ZIP"
|
|
exit 0
|
|
fi
|
|
|
|
echo "workspace-archive unavailable (http=$code) — satellite likely on old code"
|
|
[ "$FALLBACK" = "1" ] || { echo "re-run with --fallback for piecewise download, or deploy the new satellite" >&2; exit 1; }
|
|
|
|
# --- 3. Fallback: reconstruct from the per-file endpoints ------------------
|
|
echo "fallback: downloading pieces …"
|
|
STAGE="$(mktemp -d)"
|
|
trap 'rm -rf "$STAGE"; rm -f "$TMP_ZIP"' EXIT
|
|
ROOT="$STAGE/$ID"
|
|
mkdir -p "$ROOT/logs" "$ROOT/evidence"
|
|
|
|
fetch() { # <url> <outfile>
|
|
curl -sS -m 300 -o "$2.tmp" "$1" 2>/dev/null || { rm -f "$2.tmp"; return 1; }
|
|
# Proxied 404s come back as a small {"error":...} JSON body — treat as absent.
|
|
if head -c 256 "$2.tmp" 2>/dev/null | grep -q '^{"error"'; then rm -f "$2.tmp"; return 1; fi
|
|
[ -s "$2.tmp" ] && mv "$2.tmp" "$2" || { rm -f "$2.tmp"; return 1; }
|
|
}
|
|
|
|
case "$KIND" in
|
|
tasks) REPORT_PATH="fix-report"; REPORT_FILE="FixReport.md" ;;
|
|
features) REPORT_PATH="fix-report"; REPORT_FILE="ImplReport.md" ;;
|
|
security) REPORT_PATH="security-report"; REPORT_FILE="EvidenceReport.md" ;;
|
|
verifications) REPORT_PATH="report"; REPORT_FILE="Verify-Report.md" ;;
|
|
prototypes) REPORT_PATH="report"; REPORT_FILE="ProtoReport.md" ;;
|
|
esac
|
|
|
|
fetch "$BASE/api/$KIND/$ID" "$ROOT/meta.json"
|
|
fetch "$BASE/api/$KIND/$ID/$REPORT_PATH" "$ROOT/$REPORT_FILE"
|
|
[ "$KIND" = "prototypes" ] && fetch "$BASE/api/$KIND/$ID/script" "$ROOT/prototype.js"
|
|
[ "$KIND" = "verifications" ] && fetch "$BASE/api/$KIND/$ID/audit-report" "$ROOT/audit-report.json"
|
|
curl -sS -m 60 "$BASE/api/$KIND/$ID/readable-report" 2>/dev/null \
|
|
| jq -r '.html // empty' > "$ROOT/readable-report.html" 2>/dev/null || true
|
|
[ -s "$ROOT/readable-report.html" ] || rm -f "$ROOT/readable-report.html"
|
|
fetch "$BASE/api/$KIND/$ID/asking/history" "$ROOT/asking.json"
|
|
fetch "$BASE/api/$KIND/$ID/conditions" "$ROOT/conditions.json"
|
|
|
|
LOGS=0
|
|
if fetch "$BASE/api/$KIND/$ID/logs" "$ROOT/logs.json"; then
|
|
while IFS=$'\t' read -r name; do
|
|
[ -n "$name" ] || continue
|
|
fetch "$BASE/api/$KIND/$ID/logs/$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1],safe=""))' "$name")?lines=200000" "$ROOT/logs/$name"
|
|
LOGS=$((LOGS + 1))
|
|
done < <(jq -r '.files[]?.name' "$ROOT/logs.json" 2>/dev/null)
|
|
fi
|
|
# Old nodes (pre round-split logs) only expose the latest session/eval file.
|
|
[ -f "$ROOT/logs/session.log" ] || fetch "$BASE/api/$KIND/$ID/log-file?lines=200000" "$ROOT/logs/session.log"
|
|
[ -f "$ROOT/logs/eval.log" ] || fetch "$BASE/api/$KIND/$ID/eval-log-file?lines=200000" "$ROOT/logs/eval.log"
|
|
|
|
EVID=0
|
|
if fetch "$BASE/api/$KIND/$ID/evidence" "$ROOT/evidence.json"; then
|
|
while IFS=$'\t' read -r name url; do
|
|
[ -n "$name" ] || continue
|
|
if [ -n "$url" ]; then fetch "$url" "$ROOT/evidence/$name"
|
|
else fetch "$BASE/api/$KIND/$ID/evidence-file/$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1],safe=""))' "$name")" "$ROOT/evidence/$name"; fi
|
|
[ -s "$ROOT/evidence/$name" ] && EVID=$((EVID + 1))
|
|
done < <(jq -r '.files[]? | [.name, (.url // "")] | @tsv' "$ROOT/evidence.json" 2>/dev/null)
|
|
fi
|
|
[ "$EVID" -gt 0 ] || rmdir "$ROOT/evidence" 2>/dev/null || true
|
|
|
|
# Python zipfile instead of the zip CLI: the latter aborts ("Zip file
|
|
# structure invalid") on evidence names containing unusual characters, which
|
|
# record-evidence occasionally produces.
|
|
python3 - "$STAGE" "$ID" "$TMP_ZIP" <<'PY'
|
|
import os, sys, zipfile
|
|
stage, root, out = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as z:
|
|
for base, _, files in os.walk(os.path.join(stage, root)):
|
|
for f in files:
|
|
full = os.path.join(base, f)
|
|
z.write(full, os.path.relpath(full, stage))
|
|
PY
|
|
[ -s "$TMP_ZIP" ] || { echo "packing failed; pieces left in $ROOT" >&2; exit 1; }
|
|
mv "$TMP_ZIP" "$OUT_ZIP"
|
|
trap 'rm -rf "$STAGE"' EXIT
|
|
rm -rf "$STAGE"
|
|
echo "saved fallback zip (logs=$LOGS evidence=$EVID) $(du -h "$OUT_ZIP" | awk '{print $1}') -> $OUT_ZIP"
|