Files
streamline/.claude/skills/streamline-retro/fetch.sh
T
2026-11-16 17:26:44 -08:00

170 lines
8.2 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ---------------------------------------------------------------------------
# streamline-retro / fetch.sh — snapshot everything about one task id to disk.
#
# bash fetch.sh <id> [--out <dir>] [--evidence] [--lines N]
#
# Read-only: every call is a GET against the control plane's public read API
# (STREAMLINE_API, default https://streamline-internal-api.bytedance.net).
# No DB credentials, no writes. The control plane already proxies per-node
# artifacts (session log / ImplReport / evidence) for us, so one host is enough
# regardless of which satellite ran the task — EXCEPT that its log endpoints cap
# a file at 8 MB (tail). Long runs blow through that (a 4-hour UI walkthrough
# writes 30–60 MB), so per-round logs whose API copy came back short are pulled
# read-only from the executing node with `scp` (needs the `ssh <node>` alias;
# skipped with a note when unreachable). --no-ssh turns that off.
#
# Prints a machine-readable KIND=… OUT=… footer so the caller knows where the
# snapshot landed and what kind of record the id turned out to be.
# ---------------------------------------------------------------------------
set -uo pipefail
BASE="${STREAMLINE_API:-https://streamline-internal-api.bytedance.net}"
ID=""
OUT_ROOT=""
WITH_EVIDENCE=0
LINES=200000
NO_SSH=0
while [ $# -gt 0 ]; do
case "$1" in
--out) OUT_ROOT="$2"; shift 2 ;;
--evidence) WITH_EVIDENCE=1; shift ;;
--lines) LINES="$2"; shift 2 ;;
--no-ssh) NO_SSH=1; shift ;;
-*) echo "unknown flag: $1" >&2; exit 2 ;;
*) ID="$1"; shift ;;
esac
done
[ -n "$ID" ] || { echo "usage: fetch.sh <task-id> [--out DIR] [--evidence] [--lines N]" >&2; exit 2; }
# Accept a pasted URL too (…?task_id=<uuid> / …/features/<uuid>) — pull the uuid out.
if [[ "$ID" =~ ([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}) ]]; then
ID="${BASH_REMATCH[1]}"
else
echo "not a uuid: $ID" >&2; exit 2
fi
OUT_ROOT="${OUT_ROOT:-${TMPDIR:-/tmp}/streamline-retro}"
OUT="$OUT_ROOT/$ID"
mkdir -p "$OUT"
get() { curl -sS -m 60 -w '\n%{http_code}' "$BASE$1" 2>/dev/null; }
# Fetch to a file; keep the file only on HTTP 200. Echoes the status code.
fetch_to() { # <path> <outfile>
local code
code=$(curl -sS -m 120 -o "$2.tmp" -w '%{http_code}' "$BASE$1" 2>/dev/null)
if [ "$code" = "200" ] && [ -s "$2.tmp" ]; then mv "$2.tmp" "$2"; else rm -f "$2.tmp"; fi
echo "$code"
}
# --- 1. Which kind of record is this id? -----------------------------------
KIND=""
for k in features tasks security verifications prototypes; do
code=$(curl -sS -m 20 -o "$OUT/meta.json.tmp" -w '%{http_code}' "$BASE/api/$k/$ID" 2>/dev/null)
if [ "$code" = "200" ]; then KIND="$k"; mv "$OUT/meta.json.tmp" "$OUT/meta.json"; break; fi
done
rm -f "$OUT/meta.json.tmp"
if [ -z "$KIND" ]; then
echo "id not found as feature / task / security / verification / prototype on $BASE" >&2
echo "KIND=none OUT=$OUT"
exit 1
fi
# The report endpoint is the one asymmetry between kinds.
REPORT_PATH="/api/$KIND/$ID/fix-report"
REPORT_FILE="ImplReport.md"
case "$KIND" in
tasks) REPORT_FILE="FixReport.md" ;;
security) REPORT_PATH="/api/security/$ID/security-report"; REPORT_FILE="SecurityReport.md" ;;
# 验证:报告原文(UI 走查 / 用例验证的 .md)+ 面板用的结构化数据;原型:接手包。
verifications) REPORT_PATH="/api/verifications/$ID/report"; REPORT_FILE="Verify-Report.md" ;;
prototypes) REPORT_PATH="/api/prototypes/$ID/report"; REPORT_FILE="ProtoReport.md" ;;
esac
# --- 2. Split the record: big blobs become their own readable files ---------
jq 'del(.readableReportHtml)' "$OUT/meta.json" > "$OUT/meta.slim.json" 2>/dev/null \
&& mv "$OUT/meta.slim.json" "$OUT/meta.json"
jq -r '.executionPlan // empty' "$OUT/meta.json" > "$OUT/TestSpec.md"
jq -r '.description // empty' "$OUT/meta.json" > "$OUT/OriginalIntro.md"
jq -r '.reviewHistory // empty' "$OUT/meta.json" > "$OUT/review-history.json"
[ -s "$OUT/TestSpec.md" ] || rm -f "$OUT/TestSpec.md"
[ -s "$OUT/OriginalIntro.md" ] || rm -f "$OUT/OriginalIntro.md"
[ -s "$OUT/review-history.json" ] || rm -f "$OUT/review-history.json"
# --- 3. Artifacts ----------------------------------------------------------
c_report=$(fetch_to "$REPORT_PATH" "$OUT/$REPORT_FILE")
c_sess=$( fetch_to "/api/$KIND/$ID/log-file?lines=$LINES" "$OUT/session.jsonl")
c_eval=$( fetch_to "/api/$KIND/$ID/eval-log-file?lines=$LINES" "$OUT/eval.jsonl")
c_evid=$( fetch_to "/api/$KIND/$ID/evidence" "$OUT/evidence.json")
c_ask=$( fetch_to "/api/$KIND/$ID/asking/history" "$OUT/asking.json")
c_read=$( fetch_to "/api/$KIND/$ID/readable-report" "$OUT/readable-report.json")
c_audit="-"
[ "$KIND" = "verifications" ] && c_audit=$(fetch_to "/api/verifications/$ID/audit-report" "$OUT/audit-report.json")
# --- 3b. Every round's session log + step timing, full-size ---------------
# log-file above is the latest round only, and capped at 8 MB by the API. Long
# runs need every round and the whole file, so: list logs/, fetch each, and when
# the API copy is shorter than the node says the file is, scp the original.
LOGS_N=0; LOGS_SSH=0; LOGS_SHORT=""
NODE=$(jq -r '.executorNodeId // empty' "$OUT/meta.json"); NODE_ALIAS="${NODE%%.*}"
WS=$(jq -r '.workspacePath // empty' "$OUT/meta.json")
if fetch_to "/api/$KIND/$ID/logs" "$OUT/logs.json" >/dev/null && [ -s "$OUT/logs.json" ]; then
mkdir -p "$OUT/logs"
while IFS=$'\t' read -r name size; do
[ -n "$name" ] || continue
fetch_to "/api/$KIND/$ID/logs/$name?lines=$LINES" "$OUT/logs/$name" >/dev/null
LOGS_N=$((LOGS_N + 1))
got=$(wc -c < "$OUT/logs/$name" 2>/dev/null || echo 0)
if [ "${size%.*}" -gt $((got + 1024)) ]; then
if [ "$NO_SSH" = "0" ] && [ -n "$NODE_ALIAS" ] && [ -n "$WS" ] \
&& scp -q -o BatchMode=yes -o ConnectTimeout=8 "$NODE_ALIAS:$WS/logs/$name" "$OUT/logs/$name" 2>/dev/null; then
LOGS_SSH=$((LOGS_SSH + 1))
else
LOGS_SHORT="$LOGS_SHORT $name"
fi
fi
done < <(jq -r '.files[]? | select(.kind == "session" or .kind == "timing" or .kind == "runner" or .kind == "critic") | [.name, (.size|tostring)] | @tsv' "$OUT/logs.json" 2>/dev/null)
fi
# readable report is JSON-wrapped HTML — unwrap so it can be read/opened directly.
if [ -s "$OUT/readable-report.json" ]; then
jq -r '.html // empty' "$OUT/readable-report.json" > "$OUT/readable-report.html" 2>/dev/null
[ -s "$OUT/readable-report.html" ] || rm -f "$OUT/readable-report.html"
fi
# --- 4. Optional: pull evidence images so they can be looked at ------------
EVID_N=0
if [ "$WITH_EVIDENCE" = "1" ] && [ -s "$OUT/evidence.json" ]; then
mkdir -p "$OUT/evidence"
while IFS=$'\t' read -r name url; do
[ -n "$url" ] || continue
curl -sS -m 120 -o "$OUT/evidence/$name" "$url" 2>/dev/null && EVID_N=$((EVID_N + 1))
done < <(jq -r '.files[]? | select(.isImage == true) | [.name, .url] | @tsv' "$OUT/evidence.json" 2>/dev/null | head -40)
fi
# --- 5. Report what we got -------------------------------------------------
{
echo "kind : $KIND"
echo "id : $ID"
echo "title : $(jq -r '.title // "?"' "$OUT/meta.json")"
echo "status : $(jq -r '.status // "?"' "$OUT/meta.json") attempts=$(jq -r '.attempts // 0' "$OUT/meta.json") platform=$(jq -r '.platform // "?"' "$OUT/meta.json")"
echo "node : $(jq -r '.executorNodeId // "-"' "$OUT/meta.json")"
echo "branch/MR : $(jq -r '.branchName // "-"' "$OUT/meta.json") | $(jq -r '.mrUrl // "-"' "$OUT/meta.json")"
echo "failure : $(jq -r '(.failureReason // "-") | gsub("\n"; " ⏎ ") | .[0:400]' "$OUT/meta.json")"
echo
echo "artifacts (http codes; 404 = that artifact does not exist for this id):"
echo " $REPORT_FILE=$c_report session.jsonl=$c_sess eval.jsonl=$c_eval evidence.json=$c_evid asking.json=$c_ask readable=$c_read audit-report.json=$c_audit"
echo " logs/: $LOGS_N files (per round)$( [ "$LOGS_SSH" -gt 0 ] && echo ", $LOGS_SSH pulled full-size from $NODE_ALIAS via scp" )"
[ -n "$LOGS_SHORT" ] && echo " ⚠ still truncated at the API's 8 MB (node unreachable or --no-ssh):$LOGS_SHORT"
[ "$WITH_EVIDENCE" = "1" ] && echo " evidence images downloaded: $EVID_N"
echo
echo "OUT=$OUT"
echo "KIND=$KIND"
} | tee "$OUT/fetch-summary.txt"