Initial import of streamline

This commit is contained in:
2026-11-16 17:26:44 -08:00
commit 8e4aec1855
1221 changed files with 274256 additions and 0 deletions
+240
View File
@@ -0,0 +1,240 @@
#!/usr/bin/env bash
# ---------------------------------------------------------------------------
# download-task.sh — download one Streamline task's whole execution workspace
# (reports, every round's logs, evidence incl. videos, TestSpec/EffectiveSpec,
# Conditions.jsonl, images, manifests) as <id>.zip.
#
# bash scripts/download-task.sh <id> [--out DIR] [--direct] [--node HOST] [--fallback]
#
# The zip is produced ON THE SATELLITE by its workspace-archive endpoint, so
# the content matches tarring the per-task execution dir directly — minus the
# TikTok source checkout, built APK/AAB/IPA and artifacts/ (hook/XP modules +
# post-run instrumentation: jacoco, tracking/AppLog, open-audit), excluded
# server-side.
#
# Two ways to reach that endpoint:
# default via the control plane proxy /api/<kind>/<id>/workspace-archive
# (requires a deployed control plane)
# --direct straight to the home satellite's /sat/.../workspace-archive
# (only the satellite needs the new code; the node base URL is
# resolved from read-only /api/nodes, or pinned with --node;
# set CLUSTER_SECRET if the cluster requires the shared header)
#
# Requires company intranet access to the read API
# (STREAMLINE_API, default https://streamline-internal-api.bytedance.net).
# Read-only: every request is a GET.
#
# --fallback also enables a client-side reconstruction (report + logs list +
# evidence one by one + DB-side readable report) for nodes still running an
# older satellite without the workspace-archive endpoint. It cannot include
# the workspace's images/ or be byte-identical to a dir tar.
# ---------------------------------------------------------------------------
set -uo pipefail
BASE="${STREAMLINE_API:-https://streamline-internal-api.bytedance.net}"
ID=""
OUT_DIR="."
FALLBACK=0
DIRECT=0
NODE_PIN=""
while [ $# -gt 0 ]; do
case "$1" in
--out) OUT_DIR="$2"; shift 2 ;;
--fallback) FALLBACK=1; shift ;;
--direct) DIRECT=1; shift ;;
--node) NODE_PIN="$2"; shift 2 ;;
-*) echo "unknown flag: $1" >&2; exit 2 ;;
*) ID="$1"; shift ;;
esac
done
[ -n "$ID" ] || { echo "usage: download-task.sh <task-id> [--out DIR] [--fallback]" >&2; exit 2; }
# Accept pasted panel URLs too — pull the uuid out.
if [[ "$ID" =~ ([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}) ]]; then
ID="${BASH_REMATCH[1]}"
else
echo "not a uuid: $ID" >&2; exit 2
fi
mkdir -p "$OUT_DIR"
OUT_ZIP="$OUT_DIR/$ID.zip"
TMP_ZIP="$OUT_ZIP.tmp"
trap 'rm -f "$TMP_ZIP"' EXIT
# --- 1. Which track does this id belong to? --------------------------------
KIND=""
for k in tasks features security verifications prototypes; do
code=$(curl -sS -m 20 -o /dev/null -w '%{http_code}' "$BASE/api/$k/$ID" 2>/dev/null)
if [ "$code" = "200" ]; then KIND="$k"; break; fi
done
if [ -z "$KIND" ]; then
echo "id not found as task/feature/security/verification/prototype on $BASE" >&2
exit 1
fi
echo "id=$ID kind=$KIND"
# --- 2. Fast path: the satellite zips its real workspace dir ---------------
# Workspace key differs for bugs: satellites key bug dirs by meegoId||id.
META_URL="$BASE/api/$KIND/$ID"
ARCHIVE_URL="$BASE/api/$KIND/$ID/workspace-archive"
EXTRA_HEADER=()
NODE_DESC="control-plane proxy"
if [ "$DIRECT" = "1" ]; then
META_BODY="$(curl -sS -m 20 "$META_URL" 2>/dev/null || true)"
NODE_ID_RESOLVED="$(printf '%s' "$META_BODY" | jq -r '.executorNodeId // empty' 2>/dev/null)"
[ -n "$NODE_PIN" ] && NODE_ID_RESOLVED="$NODE_PIN"
if [ -z "$NODE_ID_RESOLVED" ]; then
echo "task has no executorNodeId (pre-cluster legacy record) — direct mode needs a home node; try without --direct" >&2
exit 1
fi
# --node accepts either a registry node id (resolved via /api/nodes) or a
# direct base URL (http://host:port).
if [[ "$NODE_ID_RESOLVED" == http://* || "$NODE_ID_RESOLVED" == https://* ]]; then
NODE_BASE="${NODE_ID_RESOLVED%/}"
else
NODE_BASE="$(curl -sS -m 20 "$BASE/api/nodes" 2>/dev/null \
| jq -r --arg n "$NODE_ID_RESOLVED" '(.nodes // . // [])[] | select(.id==$n) | .baseUrl' 2>/dev/null | head -1)"
if [ -z "$NODE_BASE" ]; then
echo "node $NODE_ID_RESOLVED not found in /api/nodes (offline/pruned?)" >&2
exit 1
fi
fi
case "$KIND" in
tasks)
KEY="$(printf '%s' "$META_BODY" | jq -r '.meegoId // .id' 2>/dev/null)"
ARCHIVE_URL="$NODE_BASE/sat/ws/$KEY/workspace-archive" ;;
features) ARCHIVE_URL="$NODE_BASE/sat/features/$ID/workspace-archive" ;;
security) ARCHIVE_URL="$NODE_BASE/sat/security/$ID/workspace-archive" ;;
verifications) ARCHIVE_URL="$NODE_BASE/sat/verifications/$ID/workspace-archive" ;;
prototypes) ARCHIVE_URL="$NODE_BASE/sat/prototypes/$ID/workspace-archive" ;;
esac
NODE_DESC="$NODE_ID_RESOLVED ($NODE_BASE)"
[ -n "${CLUSTER_SECRET:-}" ] && EXTRA_HEADER=(-H "x-cluster-secret: $CLUSTER_SECRET")
fi
echo "archive endpoint: $NODE_DESC"
code=$(curl -sS -m 600 ${EXTRA_HEADER[@]+"${EXTRA_HEADER[@]}"} -o "$TMP_ZIP" -w '%{http_code}' \
"$ARCHIVE_URL" 2>/dev/null || true)
is_zip() { head -c2 "$1" 2>/dev/null | grep -q $'PK'; }
if [ "$code" = "200" ] && is_zip "$TMP_ZIP"; then
# The readable report + full task record live in the control-plane DB, not in
# the satellite workspace — append them so the zip is self-contained for
# review too. Best-effort; needs local zip/unzip.
if command -v python3 >/dev/null 2>&1; then
STAGE="$(mktemp -d)"
META="$STAGE/meta.json"; HTML="$STAGE/readable-report.html"
curl -sS -m 30 -o "$META" "$BASE/api/$KIND/$ID" 2>/dev/null || true
curl -sS -m 60 "$BASE/api/$KIND/$ID/readable-report" 2>/dev/null \
| jq -r '.html // empty' > "$HTML" 2>/dev/null || true
[ -s "$META" ] || META=""
[ -s "$HTML" ] || HTML=""
if [ -n "$META" ] || [ -n "$HTML" ]; then
python3 - "$TMP_ZIP" "$META" "$HTML" <<'PY'
import os, sys, zipfile
zp, meta, html = sys.argv[1], sys.argv[2], sys.argv[3]
extras = {}
if meta: extras["meta.json"] = meta
if html: extras["readable-report.html"] = html
with zipfile.ZipFile(zp) as zin:
names = zin.namelist()
prefix = names[0].split("/", 1)[0] if names else ""
existing = set(names)
with zipfile.ZipFile(zp + ".new", "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zout:
for item in zin.infolist():
zout.writestr(item, zin.read(item.filename))
for arc, src in extras.items():
dest = f"{prefix}/{arc}" if prefix else arc
if dest not in existing:
zout.write(src, dest)
os.replace(zp + ".new", zp)
PY
fi
rm -rf "$STAGE"
fi
mv "$TMP_ZIP" "$OUT_ZIP"
trap - EXIT
echo "saved $(du -h "$OUT_ZIP" | awk '{print $1}') -> $OUT_ZIP"
exit 0
fi
echo "workspace-archive unavailable (http=$code) — satellite likely on old code"
[ "$FALLBACK" = "1" ] || { echo "re-run with --fallback for piecewise download, or deploy the new satellite" >&2; exit 1; }
# --- 3. Fallback: reconstruct from the per-file endpoints ------------------
echo "fallback: downloading pieces …"
STAGE="$(mktemp -d)"
trap 'rm -rf "$STAGE"; rm -f "$TMP_ZIP"' EXIT
ROOT="$STAGE/$ID"
mkdir -p "$ROOT/logs" "$ROOT/evidence"
fetch() { # <url> <outfile>
curl -sS -m 300 -o "$2.tmp" "$1" 2>/dev/null || { rm -f "$2.tmp"; return 1; }
# Proxied 404s come back as a small {"error":...} JSON body — treat as absent.
if head -c 256 "$2.tmp" 2>/dev/null | grep -q '^{"error"'; then rm -f "$2.tmp"; return 1; fi
[ -s "$2.tmp" ] && mv "$2.tmp" "$2" || { rm -f "$2.tmp"; return 1; }
}
case "$KIND" in
tasks) REPORT_PATH="fix-report"; REPORT_FILE="FixReport.md" ;;
features) REPORT_PATH="fix-report"; REPORT_FILE="ImplReport.md" ;;
security) REPORT_PATH="security-report"; REPORT_FILE="EvidenceReport.md" ;;
verifications) REPORT_PATH="report"; REPORT_FILE="Verify-Report.md" ;;
prototypes) REPORT_PATH="report"; REPORT_FILE="ProtoReport.md" ;;
esac
fetch "$BASE/api/$KIND/$ID" "$ROOT/meta.json"
fetch "$BASE/api/$KIND/$ID/$REPORT_PATH" "$ROOT/$REPORT_FILE"
[ "$KIND" = "prototypes" ] && fetch "$BASE/api/$KIND/$ID/script" "$ROOT/prototype.js"
[ "$KIND" = "verifications" ] && fetch "$BASE/api/$KIND/$ID/audit-report" "$ROOT/audit-report.json"
curl -sS -m 60 "$BASE/api/$KIND/$ID/readable-report" 2>/dev/null \
| jq -r '.html // empty' > "$ROOT/readable-report.html" 2>/dev/null || true
[ -s "$ROOT/readable-report.html" ] || rm -f "$ROOT/readable-report.html"
fetch "$BASE/api/$KIND/$ID/asking/history" "$ROOT/asking.json"
fetch "$BASE/api/$KIND/$ID/conditions" "$ROOT/conditions.json"
LOGS=0
if fetch "$BASE/api/$KIND/$ID/logs" "$ROOT/logs.json"; then
while IFS=$'\t' read -r name; do
[ -n "$name" ] || continue
fetch "$BASE/api/$KIND/$ID/logs/$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1],safe=""))' "$name")?lines=200000" "$ROOT/logs/$name"
LOGS=$((LOGS + 1))
done < <(jq -r '.files[]?.name' "$ROOT/logs.json" 2>/dev/null)
fi
# Old nodes (pre round-split logs) only expose the latest session/eval file.
[ -f "$ROOT/logs/session.log" ] || fetch "$BASE/api/$KIND/$ID/log-file?lines=200000" "$ROOT/logs/session.log"
[ -f "$ROOT/logs/eval.log" ] || fetch "$BASE/api/$KIND/$ID/eval-log-file?lines=200000" "$ROOT/logs/eval.log"
EVID=0
if fetch "$BASE/api/$KIND/$ID/evidence" "$ROOT/evidence.json"; then
while IFS=$'\t' read -r name url; do
[ -n "$name" ] || continue
if [ -n "$url" ]; then fetch "$url" "$ROOT/evidence/$name"
else fetch "$BASE/api/$KIND/$ID/evidence-file/$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1],safe=""))' "$name")" "$ROOT/evidence/$name"; fi
[ -s "$ROOT/evidence/$name" ] && EVID=$((EVID + 1))
done < <(jq -r '.files[]? | [.name, (.url // "")] | @tsv' "$ROOT/evidence.json" 2>/dev/null)
fi
[ "$EVID" -gt 0 ] || rmdir "$ROOT/evidence" 2>/dev/null || true
# Python zipfile instead of the zip CLI: the latter aborts ("Zip file
# structure invalid") on evidence names containing unusual characters, which
# record-evidence occasionally produces.
python3 - "$STAGE" "$ID" "$TMP_ZIP" <<'PY'
import os, sys, zipfile
stage, root, out = sys.argv[1], sys.argv[2], sys.argv[3]
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as z:
for base, _, files in os.walk(os.path.join(stage, root)):
for f in files:
full = os.path.join(base, f)
z.write(full, os.path.relpath(full, stage))
PY
[ -s "$TMP_ZIP" ] || { echo "packing failed; pieces left in $ROOT" >&2; exit 1; }
mv "$TMP_ZIP" "$OUT_ZIP"
trap 'rm -rf "$STAGE"' EXIT
rm -rf "$STAGE"
echo "saved fallback zip (logs=$LOGS evidence=$EVID) $(du -h "$OUT_ZIP" | awk '{print $1}') -> $OUT_ZIP"