Initial import of streamline
This commit is contained in:
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env bash
|
||||
# ---------------------------------------------------------------------------
|
||||
# download-task.sh — download one Streamline task's whole execution workspace
|
||||
# (reports, every round's logs, evidence incl. videos, TestSpec/EffectiveSpec,
|
||||
# Conditions.jsonl, images, manifests) as <id>.zip.
|
||||
#
|
||||
# bash scripts/download-task.sh <id> [--out DIR] [--direct] [--node HOST] [--fallback]
|
||||
#
|
||||
# The zip is produced ON THE SATELLITE by its workspace-archive endpoint, so
|
||||
# the content matches tarring the per-task execution dir directly — minus the
|
||||
# TikTok source checkout, built APK/AAB/IPA and artifacts/ (hook/XP modules +
|
||||
# post-run instrumentation: jacoco, tracking/AppLog, open-audit), excluded
|
||||
# server-side.
|
||||
#
|
||||
# Two ways to reach that endpoint:
|
||||
# default via the control plane proxy /api/<kind>/<id>/workspace-archive
|
||||
# (requires a deployed control plane)
|
||||
# --direct straight to the home satellite's /sat/.../workspace-archive
|
||||
# (only the satellite needs the new code; the node base URL is
|
||||
# resolved from read-only /api/nodes, or pinned with --node;
|
||||
# set CLUSTER_SECRET if the cluster requires the shared header)
|
||||
#
|
||||
# Requires company intranet access to the read API
|
||||
# (STREAMLINE_API, default https://streamline-internal-api.bytedance.net).
|
||||
# Read-only: every request is a GET.
|
||||
#
|
||||
# --fallback also enables a client-side reconstruction (report + logs list +
|
||||
# evidence one by one + DB-side readable report) for nodes still running an
|
||||
# older satellite without the workspace-archive endpoint. It cannot include
|
||||
# the workspace's images/ or be byte-identical to a dir tar.
|
||||
# ---------------------------------------------------------------------------
|
||||
set -uo pipefail
|
||||
|
||||
BASE="${STREAMLINE_API:-https://streamline-internal-api.bytedance.net}"
|
||||
ID=""
|
||||
OUT_DIR="."
|
||||
FALLBACK=0
|
||||
DIRECT=0
|
||||
NODE_PIN=""
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--out) OUT_DIR="$2"; shift 2 ;;
|
||||
--fallback) FALLBACK=1; shift ;;
|
||||
--direct) DIRECT=1; shift ;;
|
||||
--node) NODE_PIN="$2"; shift 2 ;;
|
||||
-*) echo "unknown flag: $1" >&2; exit 2 ;;
|
||||
*) ID="$1"; shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -n "$ID" ] || { echo "usage: download-task.sh <task-id> [--out DIR] [--fallback]" >&2; exit 2; }
|
||||
|
||||
# Accept pasted panel URLs too — pull the uuid out.
|
||||
if [[ "$ID" =~ ([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}) ]]; then
|
||||
ID="${BASH_REMATCH[1]}"
|
||||
else
|
||||
echo "not a uuid: $ID" >&2; exit 2
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
OUT_ZIP="$OUT_DIR/$ID.zip"
|
||||
TMP_ZIP="$OUT_ZIP.tmp"
|
||||
trap 'rm -f "$TMP_ZIP"' EXIT
|
||||
|
||||
# --- 1. Which track does this id belong to? --------------------------------
|
||||
KIND=""
|
||||
for k in tasks features security verifications prototypes; do
|
||||
code=$(curl -sS -m 20 -o /dev/null -w '%{http_code}' "$BASE/api/$k/$ID" 2>/dev/null)
|
||||
if [ "$code" = "200" ]; then KIND="$k"; break; fi
|
||||
done
|
||||
|
||||
if [ -z "$KIND" ]; then
|
||||
echo "id not found as task/feature/security/verification/prototype on $BASE" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "id=$ID kind=$KIND"
|
||||
|
||||
# --- 2. Fast path: the satellite zips its real workspace dir ---------------
|
||||
# Workspace key differs for bugs: satellites key bug dirs by meegoId||id.
|
||||
META_URL="$BASE/api/$KIND/$ID"
|
||||
ARCHIVE_URL="$BASE/api/$KIND/$ID/workspace-archive"
|
||||
EXTRA_HEADER=()
|
||||
NODE_DESC="control-plane proxy"
|
||||
|
||||
if [ "$DIRECT" = "1" ]; then
|
||||
META_BODY="$(curl -sS -m 20 "$META_URL" 2>/dev/null || true)"
|
||||
NODE_ID_RESOLVED="$(printf '%s' "$META_BODY" | jq -r '.executorNodeId // empty' 2>/dev/null)"
|
||||
[ -n "$NODE_PIN" ] && NODE_ID_RESOLVED="$NODE_PIN"
|
||||
if [ -z "$NODE_ID_RESOLVED" ]; then
|
||||
echo "task has no executorNodeId (pre-cluster legacy record) — direct mode needs a home node; try without --direct" >&2
|
||||
exit 1
|
||||
fi
|
||||
# --node accepts either a registry node id (resolved via /api/nodes) or a
|
||||
# direct base URL (http://host:port).
|
||||
if [[ "$NODE_ID_RESOLVED" == http://* || "$NODE_ID_RESOLVED" == https://* ]]; then
|
||||
NODE_BASE="${NODE_ID_RESOLVED%/}"
|
||||
else
|
||||
NODE_BASE="$(curl -sS -m 20 "$BASE/api/nodes" 2>/dev/null \
|
||||
| jq -r --arg n "$NODE_ID_RESOLVED" '(.nodes // . // [])[] | select(.id==$n) | .baseUrl' 2>/dev/null | head -1)"
|
||||
if [ -z "$NODE_BASE" ]; then
|
||||
echo "node $NODE_ID_RESOLVED not found in /api/nodes (offline/pruned?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
case "$KIND" in
|
||||
tasks)
|
||||
KEY="$(printf '%s' "$META_BODY" | jq -r '.meegoId // .id' 2>/dev/null)"
|
||||
ARCHIVE_URL="$NODE_BASE/sat/ws/$KEY/workspace-archive" ;;
|
||||
features) ARCHIVE_URL="$NODE_BASE/sat/features/$ID/workspace-archive" ;;
|
||||
security) ARCHIVE_URL="$NODE_BASE/sat/security/$ID/workspace-archive" ;;
|
||||
verifications) ARCHIVE_URL="$NODE_BASE/sat/verifications/$ID/workspace-archive" ;;
|
||||
prototypes) ARCHIVE_URL="$NODE_BASE/sat/prototypes/$ID/workspace-archive" ;;
|
||||
esac
|
||||
NODE_DESC="$NODE_ID_RESOLVED ($NODE_BASE)"
|
||||
[ -n "${CLUSTER_SECRET:-}" ] && EXTRA_HEADER=(-H "x-cluster-secret: $CLUSTER_SECRET")
|
||||
fi
|
||||
|
||||
echo "archive endpoint: $NODE_DESC"
|
||||
code=$(curl -sS -m 600 ${EXTRA_HEADER[@]+"${EXTRA_HEADER[@]}"} -o "$TMP_ZIP" -w '%{http_code}' \
|
||||
"$ARCHIVE_URL" 2>/dev/null || true)
|
||||
|
||||
is_zip() { head -c2 "$1" 2>/dev/null | grep -q $'PK'; }
|
||||
|
||||
if [ "$code" = "200" ] && is_zip "$TMP_ZIP"; then
|
||||
# The readable report + full task record live in the control-plane DB, not in
|
||||
# the satellite workspace — append them so the zip is self-contained for
|
||||
# review too. Best-effort; needs local zip/unzip.
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
STAGE="$(mktemp -d)"
|
||||
META="$STAGE/meta.json"; HTML="$STAGE/readable-report.html"
|
||||
curl -sS -m 30 -o "$META" "$BASE/api/$KIND/$ID" 2>/dev/null || true
|
||||
curl -sS -m 60 "$BASE/api/$KIND/$ID/readable-report" 2>/dev/null \
|
||||
| jq -r '.html // empty' > "$HTML" 2>/dev/null || true
|
||||
[ -s "$META" ] || META=""
|
||||
[ -s "$HTML" ] || HTML=""
|
||||
if [ -n "$META" ] || [ -n "$HTML" ]; then
|
||||
python3 - "$TMP_ZIP" "$META" "$HTML" <<'PY'
|
||||
import os, sys, zipfile
|
||||
zp, meta, html = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
extras = {}
|
||||
if meta: extras["meta.json"] = meta
|
||||
if html: extras["readable-report.html"] = html
|
||||
with zipfile.ZipFile(zp) as zin:
|
||||
names = zin.namelist()
|
||||
prefix = names[0].split("/", 1)[0] if names else ""
|
||||
existing = set(names)
|
||||
with zipfile.ZipFile(zp + ".new", "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zout:
|
||||
for item in zin.infolist():
|
||||
zout.writestr(item, zin.read(item.filename))
|
||||
for arc, src in extras.items():
|
||||
dest = f"{prefix}/{arc}" if prefix else arc
|
||||
if dest not in existing:
|
||||
zout.write(src, dest)
|
||||
os.replace(zp + ".new", zp)
|
||||
PY
|
||||
fi
|
||||
rm -rf "$STAGE"
|
||||
fi
|
||||
mv "$TMP_ZIP" "$OUT_ZIP"
|
||||
trap - EXIT
|
||||
echo "saved $(du -h "$OUT_ZIP" | awk '{print $1}') -> $OUT_ZIP"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "workspace-archive unavailable (http=$code) — satellite likely on old code"
|
||||
[ "$FALLBACK" = "1" ] || { echo "re-run with --fallback for piecewise download, or deploy the new satellite" >&2; exit 1; }
|
||||
|
||||
# --- 3. Fallback: reconstruct from the per-file endpoints ------------------
|
||||
echo "fallback: downloading pieces …"
|
||||
STAGE="$(mktemp -d)"
|
||||
trap 'rm -rf "$STAGE"; rm -f "$TMP_ZIP"' EXIT
|
||||
ROOT="$STAGE/$ID"
|
||||
mkdir -p "$ROOT/logs" "$ROOT/evidence"
|
||||
|
||||
fetch() { # <url> <outfile>
|
||||
curl -sS -m 300 -o "$2.tmp" "$1" 2>/dev/null || { rm -f "$2.tmp"; return 1; }
|
||||
# Proxied 404s come back as a small {"error":...} JSON body — treat as absent.
|
||||
if head -c 256 "$2.tmp" 2>/dev/null | grep -q '^{"error"'; then rm -f "$2.tmp"; return 1; fi
|
||||
[ -s "$2.tmp" ] && mv "$2.tmp" "$2" || { rm -f "$2.tmp"; return 1; }
|
||||
}
|
||||
|
||||
case "$KIND" in
|
||||
tasks) REPORT_PATH="fix-report"; REPORT_FILE="FixReport.md" ;;
|
||||
features) REPORT_PATH="fix-report"; REPORT_FILE="ImplReport.md" ;;
|
||||
security) REPORT_PATH="security-report"; REPORT_FILE="EvidenceReport.md" ;;
|
||||
verifications) REPORT_PATH="report"; REPORT_FILE="Verify-Report.md" ;;
|
||||
prototypes) REPORT_PATH="report"; REPORT_FILE="ProtoReport.md" ;;
|
||||
esac
|
||||
|
||||
fetch "$BASE/api/$KIND/$ID" "$ROOT/meta.json"
|
||||
fetch "$BASE/api/$KIND/$ID/$REPORT_PATH" "$ROOT/$REPORT_FILE"
|
||||
[ "$KIND" = "prototypes" ] && fetch "$BASE/api/$KIND/$ID/script" "$ROOT/prototype.js"
|
||||
[ "$KIND" = "verifications" ] && fetch "$BASE/api/$KIND/$ID/audit-report" "$ROOT/audit-report.json"
|
||||
curl -sS -m 60 "$BASE/api/$KIND/$ID/readable-report" 2>/dev/null \
|
||||
| jq -r '.html // empty' > "$ROOT/readable-report.html" 2>/dev/null || true
|
||||
[ -s "$ROOT/readable-report.html" ] || rm -f "$ROOT/readable-report.html"
|
||||
fetch "$BASE/api/$KIND/$ID/asking/history" "$ROOT/asking.json"
|
||||
fetch "$BASE/api/$KIND/$ID/conditions" "$ROOT/conditions.json"
|
||||
|
||||
LOGS=0
|
||||
if fetch "$BASE/api/$KIND/$ID/logs" "$ROOT/logs.json"; then
|
||||
while IFS=$'\t' read -r name; do
|
||||
[ -n "$name" ] || continue
|
||||
fetch "$BASE/api/$KIND/$ID/logs/$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1],safe=""))' "$name")?lines=200000" "$ROOT/logs/$name"
|
||||
LOGS=$((LOGS + 1))
|
||||
done < <(jq -r '.files[]?.name' "$ROOT/logs.json" 2>/dev/null)
|
||||
fi
|
||||
# Old nodes (pre round-split logs) only expose the latest session/eval file.
|
||||
[ -f "$ROOT/logs/session.log" ] || fetch "$BASE/api/$KIND/$ID/log-file?lines=200000" "$ROOT/logs/session.log"
|
||||
[ -f "$ROOT/logs/eval.log" ] || fetch "$BASE/api/$KIND/$ID/eval-log-file?lines=200000" "$ROOT/logs/eval.log"
|
||||
|
||||
EVID=0
|
||||
if fetch "$BASE/api/$KIND/$ID/evidence" "$ROOT/evidence.json"; then
|
||||
while IFS=$'\t' read -r name url; do
|
||||
[ -n "$name" ] || continue
|
||||
if [ -n "$url" ]; then fetch "$url" "$ROOT/evidence/$name"
|
||||
else fetch "$BASE/api/$KIND/$ID/evidence-file/$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1],safe=""))' "$name")" "$ROOT/evidence/$name"; fi
|
||||
[ -s "$ROOT/evidence/$name" ] && EVID=$((EVID + 1))
|
||||
done < <(jq -r '.files[]? | [.name, (.url // "")] | @tsv' "$ROOT/evidence.json" 2>/dev/null)
|
||||
fi
|
||||
[ "$EVID" -gt 0 ] || rmdir "$ROOT/evidence" 2>/dev/null || true
|
||||
|
||||
# Python zipfile instead of the zip CLI: the latter aborts ("Zip file
|
||||
# structure invalid") on evidence names containing unusual characters, which
|
||||
# record-evidence occasionally produces.
|
||||
python3 - "$STAGE" "$ID" "$TMP_ZIP" <<'PY'
|
||||
import os, sys, zipfile
|
||||
stage, root, out = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as z:
|
||||
for base, _, files in os.walk(os.path.join(stage, root)):
|
||||
for f in files:
|
||||
full = os.path.join(base, f)
|
||||
z.write(full, os.path.relpath(full, stage))
|
||||
PY
|
||||
[ -s "$TMP_ZIP" ] || { echo "packing failed; pieces left in $ROOT" >&2; exit 1; }
|
||||
mv "$TMP_ZIP" "$OUT_ZIP"
|
||||
trap 'rm -rf "$STAGE"' EXIT
|
||||
rm -rf "$STAGE"
|
||||
echo "saved fallback zip (logs=$LOGS evidence=$EVID) $(du -h "$OUT_ZIP" | awk '{print $1}') -> $OUT_ZIP"
|
||||
Reference in New Issue
Block a user