#!/usr/bin/env bash # --------------------------------------------------------------------------- # streamline-retro / fetch.sh — snapshot everything about one task id to disk. # # bash fetch.sh [--out ] [--evidence] [--lines N] # # Read-only: every call is a GET against the control plane's public read API # (STREAMLINE_API, default https://streamline-internal-api.bytedance.net). # No DB credentials, no writes. The control plane already proxies per-node # artifacts (session log / ImplReport / evidence) for us, so one host is enough # regardless of which satellite ran the task — EXCEPT that its log endpoints cap # a file at 8 MB (tail). Long runs blow through that (a 4-hour UI walkthrough # writes 30–60 MB), so per-round logs whose API copy came back short are pulled # read-only from the executing node with `scp` (needs the `ssh ` alias; # skipped with a note when unreachable). --no-ssh turns that off. # # Prints a machine-readable KIND=… OUT=… footer so the caller knows where the # snapshot landed and what kind of record the id turned out to be. # --------------------------------------------------------------------------- set -uo pipefail BASE="${STREAMLINE_API:-https://streamline-internal-api.bytedance.net}" ID="" OUT_ROOT="" WITH_EVIDENCE=0 LINES=200000 NO_SSH=0 while [ $# -gt 0 ]; do case "$1" in --out) OUT_ROOT="$2"; shift 2 ;; --evidence) WITH_EVIDENCE=1; shift ;; --lines) LINES="$2"; shift 2 ;; --no-ssh) NO_SSH=1; shift ;; -*) echo "unknown flag: $1" >&2; exit 2 ;; *) ID="$1"; shift ;; esac done [ -n "$ID" ] || { echo "usage: fetch.sh [--out DIR] [--evidence] [--lines N]" >&2; exit 2; } # Accept a pasted URL too (…?task_id= / …/features/) — pull the uuid out. if [[ "$ID" =~ ([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}) ]]; then ID="${BASH_REMATCH[1]}" else echo "not a uuid: $ID" >&2; exit 2 fi OUT_ROOT="${OUT_ROOT:-${TMPDIR:-/tmp}/streamline-retro}" OUT="$OUT_ROOT/$ID" mkdir -p "$OUT" get() { curl -sS -m 60 -w '\n%{http_code}' "$BASE$1" 2>/dev/null; } # Fetch to a file; keep the file only on HTTP 200. Echoes the status code. fetch_to() { # local code code=$(curl -sS -m 120 -o "$2.tmp" -w '%{http_code}' "$BASE$1" 2>/dev/null) if [ "$code" = "200" ] && [ -s "$2.tmp" ]; then mv "$2.tmp" "$2"; else rm -f "$2.tmp"; fi echo "$code" } # --- 1. Which kind of record is this id? ----------------------------------- KIND="" for k in features tasks security verifications prototypes; do code=$(curl -sS -m 20 -o "$OUT/meta.json.tmp" -w '%{http_code}' "$BASE/api/$k/$ID" 2>/dev/null) if [ "$code" = "200" ]; then KIND="$k"; mv "$OUT/meta.json.tmp" "$OUT/meta.json"; break; fi done rm -f "$OUT/meta.json.tmp" if [ -z "$KIND" ]; then echo "id not found as feature / task / security / verification / prototype on $BASE" >&2 echo "KIND=none OUT=$OUT" exit 1 fi # The report endpoint is the one asymmetry between kinds. REPORT_PATH="/api/$KIND/$ID/fix-report" REPORT_FILE="ImplReport.md" case "$KIND" in tasks) REPORT_FILE="FixReport.md" ;; security) REPORT_PATH="/api/security/$ID/security-report"; REPORT_FILE="SecurityReport.md" ;; # 验证:报告原文(UI 走查 / 用例验证的 .md)+ 面板用的结构化数据;原型:接手包。 verifications) REPORT_PATH="/api/verifications/$ID/report"; REPORT_FILE="Verify-Report.md" ;; prototypes) REPORT_PATH="/api/prototypes/$ID/report"; REPORT_FILE="ProtoReport.md" ;; esac # --- 2. Split the record: big blobs become their own readable files --------- jq 'del(.readableReportHtml)' "$OUT/meta.json" > "$OUT/meta.slim.json" 2>/dev/null \ && mv "$OUT/meta.slim.json" "$OUT/meta.json" jq -r '.executionPlan // empty' "$OUT/meta.json" > "$OUT/TestSpec.md" jq -r '.description // empty' "$OUT/meta.json" > "$OUT/OriginalIntro.md" jq -r '.reviewHistory // empty' "$OUT/meta.json" > "$OUT/review-history.json" [ -s "$OUT/TestSpec.md" ] || rm -f "$OUT/TestSpec.md" [ -s "$OUT/OriginalIntro.md" ] || rm -f "$OUT/OriginalIntro.md" [ -s "$OUT/review-history.json" ] || rm -f "$OUT/review-history.json" # --- 3. Artifacts ---------------------------------------------------------- c_report=$(fetch_to "$REPORT_PATH" "$OUT/$REPORT_FILE") c_sess=$( fetch_to "/api/$KIND/$ID/log-file?lines=$LINES" "$OUT/session.jsonl") c_eval=$( fetch_to "/api/$KIND/$ID/eval-log-file?lines=$LINES" "$OUT/eval.jsonl") c_evid=$( fetch_to "/api/$KIND/$ID/evidence" "$OUT/evidence.json") c_ask=$( fetch_to "/api/$KIND/$ID/asking/history" "$OUT/asking.json") c_read=$( fetch_to "/api/$KIND/$ID/readable-report" "$OUT/readable-report.json") c_audit="-" [ "$KIND" = "verifications" ] && c_audit=$(fetch_to "/api/verifications/$ID/audit-report" "$OUT/audit-report.json") # --- 3b. Every round's session log + step timing, full-size --------------- # log-file above is the latest round only, and capped at 8 MB by the API. Long # runs need every round and the whole file, so: list logs/, fetch each, and when # the API copy is shorter than the node says the file is, scp the original. LOGS_N=0; LOGS_SSH=0; LOGS_SHORT="" NODE=$(jq -r '.executorNodeId // empty' "$OUT/meta.json"); NODE_ALIAS="${NODE%%.*}" WS=$(jq -r '.workspacePath // empty' "$OUT/meta.json") if fetch_to "/api/$KIND/$ID/logs" "$OUT/logs.json" >/dev/null && [ -s "$OUT/logs.json" ]; then mkdir -p "$OUT/logs" while IFS=$'\t' read -r name size; do [ -n "$name" ] || continue fetch_to "/api/$KIND/$ID/logs/$name?lines=$LINES" "$OUT/logs/$name" >/dev/null LOGS_N=$((LOGS_N + 1)) got=$(wc -c < "$OUT/logs/$name" 2>/dev/null || echo 0) if [ "${size%.*}" -gt $((got + 1024)) ]; then if [ "$NO_SSH" = "0" ] && [ -n "$NODE_ALIAS" ] && [ -n "$WS" ] \ && scp -q -o BatchMode=yes -o ConnectTimeout=8 "$NODE_ALIAS:$WS/logs/$name" "$OUT/logs/$name" 2>/dev/null; then LOGS_SSH=$((LOGS_SSH + 1)) else LOGS_SHORT="$LOGS_SHORT $name" fi fi done < <(jq -r '.files[]? | select(.kind == "session" or .kind == "timing" or .kind == "runner" or .kind == "critic") | [.name, (.size|tostring)] | @tsv' "$OUT/logs.json" 2>/dev/null) fi # readable report is JSON-wrapped HTML — unwrap so it can be read/opened directly. if [ -s "$OUT/readable-report.json" ]; then jq -r '.html // empty' "$OUT/readable-report.json" > "$OUT/readable-report.html" 2>/dev/null [ -s "$OUT/readable-report.html" ] || rm -f "$OUT/readable-report.html" fi # --- 4. Optional: pull evidence images so they can be looked at ------------ EVID_N=0 if [ "$WITH_EVIDENCE" = "1" ] && [ -s "$OUT/evidence.json" ]; then mkdir -p "$OUT/evidence" while IFS=$'\t' read -r name url; do [ -n "$url" ] || continue curl -sS -m 120 -o "$OUT/evidence/$name" "$url" 2>/dev/null && EVID_N=$((EVID_N + 1)) done < <(jq -r '.files[]? | select(.isImage == true) | [.name, .url] | @tsv' "$OUT/evidence.json" 2>/dev/null | head -40) fi # --- 5. Report what we got ------------------------------------------------- { echo "kind : $KIND" echo "id : $ID" echo "title : $(jq -r '.title // "?"' "$OUT/meta.json")" echo "status : $(jq -r '.status // "?"' "$OUT/meta.json") attempts=$(jq -r '.attempts // 0' "$OUT/meta.json") platform=$(jq -r '.platform // "?"' "$OUT/meta.json")" echo "node : $(jq -r '.executorNodeId // "-"' "$OUT/meta.json")" echo "branch/MR : $(jq -r '.branchName // "-"' "$OUT/meta.json") | $(jq -r '.mrUrl // "-"' "$OUT/meta.json")" echo "failure : $(jq -r '(.failureReason // "-") | gsub("\n"; " ⏎ ") | .[0:400]' "$OUT/meta.json")" echo echo "artifacts (http codes; 404 = that artifact does not exist for this id):" echo " $REPORT_FILE=$c_report session.jsonl=$c_sess eval.jsonl=$c_eval evidence.json=$c_evid asking.json=$c_ask readable=$c_read audit-report.json=$c_audit" echo " logs/: $LOGS_N files (per round)$( [ "$LOGS_SSH" -gt 0 ] && echo ", $LOGS_SSH pulled full-size from $NODE_ALIAS via scp" )" [ -n "$LOGS_SHORT" ] && echo " ⚠ still truncated at the API's 8 MB (node unreachable or --no-ssh):$LOGS_SHORT" [ "$WITH_EVIDENCE" = "1" ] && echo " evidence images downloaded: $EVID_N" echo echo "OUT=$OUT" echo "KIND=$KIND" } | tee "$OUT/fetch-summary.txt"