170 lines
8.2 KiB
Bash
170 lines
8.2 KiB
Bash
#!/usr/bin/env bash
|
||
# ---------------------------------------------------------------------------
|
||
# streamline-retro / fetch.sh — snapshot everything about one task id to disk.
|
||
#
|
||
# bash fetch.sh <id> [--out <dir>] [--evidence] [--lines N]
|
||
#
|
||
# Read-only: every call is a GET against the control plane's public read API
|
||
# (STREAMLINE_API, default https://streamline-internal-api.bytedance.net).
|
||
# No DB credentials, no writes. The control plane already proxies per-node
|
||
# artifacts (session log / ImplReport / evidence) for us, so one host is enough
|
||
# regardless of which satellite ran the task — EXCEPT that its log endpoints cap
|
||
# a file at 8 MB (tail). Long runs blow through that (a 4-hour UI walkthrough
|
||
# writes 30–60 MB), so per-round logs whose API copy came back short are pulled
|
||
# read-only from the executing node with `scp` (needs the `ssh <node>` alias;
|
||
# skipped with a note when unreachable). --no-ssh turns that off.
|
||
#
|
||
# Prints a machine-readable KIND=… OUT=… footer so the caller knows where the
|
||
# snapshot landed and what kind of record the id turned out to be.
|
||
# ---------------------------------------------------------------------------
|
||
set -uo pipefail
|
||
|
||
BASE="${STREAMLINE_API:-https://streamline-internal-api.bytedance.net}"
|
||
ID=""
|
||
OUT_ROOT=""
|
||
WITH_EVIDENCE=0
|
||
LINES=200000
|
||
NO_SSH=0
|
||
|
||
while [ $# -gt 0 ]; do
|
||
case "$1" in
|
||
--out) OUT_ROOT="$2"; shift 2 ;;
|
||
--evidence) WITH_EVIDENCE=1; shift ;;
|
||
--lines) LINES="$2"; shift 2 ;;
|
||
--no-ssh) NO_SSH=1; shift ;;
|
||
-*) echo "unknown flag: $1" >&2; exit 2 ;;
|
||
*) ID="$1"; shift ;;
|
||
esac
|
||
done
|
||
|
||
[ -n "$ID" ] || { echo "usage: fetch.sh <task-id> [--out DIR] [--evidence] [--lines N]" >&2; exit 2; }
|
||
|
||
# Accept a pasted URL too (…?task_id=<uuid> / …/features/<uuid>) — pull the uuid out.
|
||
if [[ "$ID" =~ ([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}) ]]; then
|
||
ID="${BASH_REMATCH[1]}"
|
||
else
|
||
echo "not a uuid: $ID" >&2; exit 2
|
||
fi
|
||
|
||
OUT_ROOT="${OUT_ROOT:-${TMPDIR:-/tmp}/streamline-retro}"
|
||
OUT="$OUT_ROOT/$ID"
|
||
mkdir -p "$OUT"
|
||
|
||
get() { curl -sS -m 60 -w '\n%{http_code}' "$BASE$1" 2>/dev/null; }
|
||
|
||
# Fetch to a file; keep the file only on HTTP 200. Echoes the status code.
|
||
fetch_to() { # <path> <outfile>
|
||
local code
|
||
code=$(curl -sS -m 120 -o "$2.tmp" -w '%{http_code}' "$BASE$1" 2>/dev/null)
|
||
if [ "$code" = "200" ] && [ -s "$2.tmp" ]; then mv "$2.tmp" "$2"; else rm -f "$2.tmp"; fi
|
||
echo "$code"
|
||
}
|
||
|
||
# --- 1. Which kind of record is this id? -----------------------------------
|
||
KIND=""
|
||
for k in features tasks security verifications prototypes; do
|
||
code=$(curl -sS -m 20 -o "$OUT/meta.json.tmp" -w '%{http_code}' "$BASE/api/$k/$ID" 2>/dev/null)
|
||
if [ "$code" = "200" ]; then KIND="$k"; mv "$OUT/meta.json.tmp" "$OUT/meta.json"; break; fi
|
||
done
|
||
rm -f "$OUT/meta.json.tmp"
|
||
|
||
if [ -z "$KIND" ]; then
|
||
echo "id not found as feature / task / security / verification / prototype on $BASE" >&2
|
||
echo "KIND=none OUT=$OUT"
|
||
exit 1
|
||
fi
|
||
|
||
# The report endpoint is the one asymmetry between kinds.
|
||
REPORT_PATH="/api/$KIND/$ID/fix-report"
|
||
REPORT_FILE="ImplReport.md"
|
||
case "$KIND" in
|
||
tasks) REPORT_FILE="FixReport.md" ;;
|
||
security) REPORT_PATH="/api/security/$ID/security-report"; REPORT_FILE="SecurityReport.md" ;;
|
||
# 验证:报告原文(UI 走查 / 用例验证的 .md)+ 面板用的结构化数据;原型:接手包。
|
||
verifications) REPORT_PATH="/api/verifications/$ID/report"; REPORT_FILE="Verify-Report.md" ;;
|
||
prototypes) REPORT_PATH="/api/prototypes/$ID/report"; REPORT_FILE="ProtoReport.md" ;;
|
||
esac
|
||
|
||
# --- 2. Split the record: big blobs become their own readable files ---------
|
||
jq 'del(.readableReportHtml)' "$OUT/meta.json" > "$OUT/meta.slim.json" 2>/dev/null \
|
||
&& mv "$OUT/meta.slim.json" "$OUT/meta.json"
|
||
|
||
jq -r '.executionPlan // empty' "$OUT/meta.json" > "$OUT/TestSpec.md"
|
||
jq -r '.description // empty' "$OUT/meta.json" > "$OUT/OriginalIntro.md"
|
||
jq -r '.reviewHistory // empty' "$OUT/meta.json" > "$OUT/review-history.json"
|
||
[ -s "$OUT/TestSpec.md" ] || rm -f "$OUT/TestSpec.md"
|
||
[ -s "$OUT/OriginalIntro.md" ] || rm -f "$OUT/OriginalIntro.md"
|
||
[ -s "$OUT/review-history.json" ] || rm -f "$OUT/review-history.json"
|
||
|
||
# --- 3. Artifacts ----------------------------------------------------------
|
||
c_report=$(fetch_to "$REPORT_PATH" "$OUT/$REPORT_FILE")
|
||
c_sess=$( fetch_to "/api/$KIND/$ID/log-file?lines=$LINES" "$OUT/session.jsonl")
|
||
c_eval=$( fetch_to "/api/$KIND/$ID/eval-log-file?lines=$LINES" "$OUT/eval.jsonl")
|
||
c_evid=$( fetch_to "/api/$KIND/$ID/evidence" "$OUT/evidence.json")
|
||
c_ask=$( fetch_to "/api/$KIND/$ID/asking/history" "$OUT/asking.json")
|
||
c_read=$( fetch_to "/api/$KIND/$ID/readable-report" "$OUT/readable-report.json")
|
||
|
||
c_audit="-"
|
||
[ "$KIND" = "verifications" ] && c_audit=$(fetch_to "/api/verifications/$ID/audit-report" "$OUT/audit-report.json")
|
||
|
||
# --- 3b. Every round's session log + step timing, full-size ---------------
|
||
# log-file above is the latest round only, and capped at 8 MB by the API. Long
|
||
# runs need every round and the whole file, so: list logs/, fetch each, and when
|
||
# the API copy is shorter than the node says the file is, scp the original.
|
||
LOGS_N=0; LOGS_SSH=0; LOGS_SHORT=""
|
||
NODE=$(jq -r '.executorNodeId // empty' "$OUT/meta.json"); NODE_ALIAS="${NODE%%.*}"
|
||
WS=$(jq -r '.workspacePath // empty' "$OUT/meta.json")
|
||
if fetch_to "/api/$KIND/$ID/logs" "$OUT/logs.json" >/dev/null && [ -s "$OUT/logs.json" ]; then
|
||
mkdir -p "$OUT/logs"
|
||
while IFS=$'\t' read -r name size; do
|
||
[ -n "$name" ] || continue
|
||
fetch_to "/api/$KIND/$ID/logs/$name?lines=$LINES" "$OUT/logs/$name" >/dev/null
|
||
LOGS_N=$((LOGS_N + 1))
|
||
got=$(wc -c < "$OUT/logs/$name" 2>/dev/null || echo 0)
|
||
if [ "${size%.*}" -gt $((got + 1024)) ]; then
|
||
if [ "$NO_SSH" = "0" ] && [ -n "$NODE_ALIAS" ] && [ -n "$WS" ] \
|
||
&& scp -q -o BatchMode=yes -o ConnectTimeout=8 "$NODE_ALIAS:$WS/logs/$name" "$OUT/logs/$name" 2>/dev/null; then
|
||
LOGS_SSH=$((LOGS_SSH + 1))
|
||
else
|
||
LOGS_SHORT="$LOGS_SHORT $name"
|
||
fi
|
||
fi
|
||
done < <(jq -r '.files[]? | select(.kind == "session" or .kind == "timing" or .kind == "runner" or .kind == "critic") | [.name, (.size|tostring)] | @tsv' "$OUT/logs.json" 2>/dev/null)
|
||
fi
|
||
|
||
# readable report is JSON-wrapped HTML — unwrap so it can be read/opened directly.
|
||
if [ -s "$OUT/readable-report.json" ]; then
|
||
jq -r '.html // empty' "$OUT/readable-report.json" > "$OUT/readable-report.html" 2>/dev/null
|
||
[ -s "$OUT/readable-report.html" ] || rm -f "$OUT/readable-report.html"
|
||
fi
|
||
|
||
# --- 4. Optional: pull evidence images so they can be looked at ------------
|
||
EVID_N=0
|
||
if [ "$WITH_EVIDENCE" = "1" ] && [ -s "$OUT/evidence.json" ]; then
|
||
mkdir -p "$OUT/evidence"
|
||
while IFS=$'\t' read -r name url; do
|
||
[ -n "$url" ] || continue
|
||
curl -sS -m 120 -o "$OUT/evidence/$name" "$url" 2>/dev/null && EVID_N=$((EVID_N + 1))
|
||
done < <(jq -r '.files[]? | select(.isImage == true) | [.name, .url] | @tsv' "$OUT/evidence.json" 2>/dev/null | head -40)
|
||
fi
|
||
|
||
# --- 5. Report what we got -------------------------------------------------
|
||
{
|
||
echo "kind : $KIND"
|
||
echo "id : $ID"
|
||
echo "title : $(jq -r '.title // "?"' "$OUT/meta.json")"
|
||
echo "status : $(jq -r '.status // "?"' "$OUT/meta.json") attempts=$(jq -r '.attempts // 0' "$OUT/meta.json") platform=$(jq -r '.platform // "?"' "$OUT/meta.json")"
|
||
echo "node : $(jq -r '.executorNodeId // "-"' "$OUT/meta.json")"
|
||
echo "branch/MR : $(jq -r '.branchName // "-"' "$OUT/meta.json") | $(jq -r '.mrUrl // "-"' "$OUT/meta.json")"
|
||
echo "failure : $(jq -r '(.failureReason // "-") | gsub("\n"; " ⏎ ") | .[0:400]' "$OUT/meta.json")"
|
||
echo
|
||
echo "artifacts (http codes; 404 = that artifact does not exist for this id):"
|
||
echo " $REPORT_FILE=$c_report session.jsonl=$c_sess eval.jsonl=$c_eval evidence.json=$c_evid asking.json=$c_ask readable=$c_read audit-report.json=$c_audit"
|
||
echo " logs/: $LOGS_N files (per round)$( [ "$LOGS_SSH" -gt 0 ] && echo ", $LOGS_SSH pulled full-size from $NODE_ALIAS via scp" )"
|
||
[ -n "$LOGS_SHORT" ] && echo " ⚠ still truncated at the API's 8 MB (node unreachable or --no-ssh):$LOGS_SHORT"
|
||
[ "$WITH_EVIDENCE" = "1" ] && echo " evidence images downloaded: $EVID_N"
|
||
echo
|
||
echo "OUT=$OUT"
|
||
echo "KIND=$KIND"
|
||
} | tee "$OUT/fetch-summary.txt"
|