Initial import of streamline

This commit is contained in:
2026-11-16 17:26:44 -08:00
commit 8e4aec1855
1221 changed files with 274256 additions and 0 deletions
+169
View File
@@ -0,0 +1,169 @@
#!/usr/bin/env bash
# ---------------------------------------------------------------------------
# streamline-retro / fetch.sh — snapshot everything about one task id to disk.
#
# bash fetch.sh <id> [--out <dir>] [--evidence] [--lines N]
#
# Read-only: every call is a GET against the control plane's public read API
# (STREAMLINE_API, default https://streamline-internal-api.bytedance.net).
# No DB credentials, no writes. The control plane already proxies per-node
# artifacts (session log / ImplReport / evidence) for us, so one host is enough
# regardless of which satellite ran the task — EXCEPT that its log endpoints cap
# a file at 8 MB (tail). Long runs blow through that (a 4-hour UI walkthrough
# writes 30–60 MB), so per-round logs whose API copy came back short are pulled
# read-only from the executing node with `scp` (needs the `ssh <node>` alias;
# skipped with a note when unreachable). --no-ssh turns that off.
#
# Prints a machine-readable KIND=… OUT=… footer so the caller knows where the
# snapshot landed and what kind of record the id turned out to be.
# ---------------------------------------------------------------------------
set -uo pipefail
BASE="${STREAMLINE_API:-https://streamline-internal-api.bytedance.net}"
ID=""
OUT_ROOT=""
WITH_EVIDENCE=0
LINES=200000
NO_SSH=0
while [ $# -gt 0 ]; do
case "$1" in
--out) OUT_ROOT="$2"; shift 2 ;;
--evidence) WITH_EVIDENCE=1; shift ;;
--lines) LINES="$2"; shift 2 ;;
--no-ssh) NO_SSH=1; shift ;;
-*) echo "unknown flag: $1" >&2; exit 2 ;;
*) ID="$1"; shift ;;
esac
done
[ -n "$ID" ] || { echo "usage: fetch.sh <task-id> [--out DIR] [--evidence] [--lines N]" >&2; exit 2; }
# Accept a pasted URL too (…?task_id=<uuid> / …/features/<uuid>) — pull the uuid out.
if [[ "$ID" =~ ([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}) ]]; then
ID="${BASH_REMATCH[1]}"
else
echo "not a uuid: $ID" >&2; exit 2
fi
OUT_ROOT="${OUT_ROOT:-${TMPDIR:-/tmp}/streamline-retro}"
OUT="$OUT_ROOT/$ID"
mkdir -p "$OUT"
get() { curl -sS -m 60 -w '\n%{http_code}' "$BASE$1" 2>/dev/null; }
# Fetch to a file; keep the file only on HTTP 200. Echoes the status code.
fetch_to() { # <path> <outfile>
local code
code=$(curl -sS -m 120 -o "$2.tmp" -w '%{http_code}' "$BASE$1" 2>/dev/null)
if [ "$code" = "200" ] && [ -s "$2.tmp" ]; then mv "$2.tmp" "$2"; else rm -f "$2.tmp"; fi
echo "$code"
}
# --- 1. Which kind of record is this id? -----------------------------------
KIND=""
for k in features tasks security verifications prototypes; do
code=$(curl -sS -m 20 -o "$OUT/meta.json.tmp" -w '%{http_code}' "$BASE/api/$k/$ID" 2>/dev/null)
if [ "$code" = "200" ]; then KIND="$k"; mv "$OUT/meta.json.tmp" "$OUT/meta.json"; break; fi
done
rm -f "$OUT/meta.json.tmp"
if [ -z "$KIND" ]; then
echo "id not found as feature / task / security / verification / prototype on $BASE" >&2
echo "KIND=none OUT=$OUT"
exit 1
fi
# The report endpoint is the one asymmetry between kinds.
REPORT_PATH="/api/$KIND/$ID/fix-report"
REPORT_FILE="ImplReport.md"
case "$KIND" in
tasks) REPORT_FILE="FixReport.md" ;;
security) REPORT_PATH="/api/security/$ID/security-report"; REPORT_FILE="SecurityReport.md" ;;
# 验证:报告原文(UI 走查 / 用例验证的 .md)+ 面板用的结构化数据;原型:接手包。
verifications) REPORT_PATH="/api/verifications/$ID/report"; REPORT_FILE="Verify-Report.md" ;;
prototypes) REPORT_PATH="/api/prototypes/$ID/report"; REPORT_FILE="ProtoReport.md" ;;
esac
# --- 2. Split the record: big blobs become their own readable files ---------
jq 'del(.readableReportHtml)' "$OUT/meta.json" > "$OUT/meta.slim.json" 2>/dev/null \
&& mv "$OUT/meta.slim.json" "$OUT/meta.json"
jq -r '.executionPlan // empty' "$OUT/meta.json" > "$OUT/TestSpec.md"
jq -r '.description // empty' "$OUT/meta.json" > "$OUT/OriginalIntro.md"
jq -r '.reviewHistory // empty' "$OUT/meta.json" > "$OUT/review-history.json"
[ -s "$OUT/TestSpec.md" ] || rm -f "$OUT/TestSpec.md"
[ -s "$OUT/OriginalIntro.md" ] || rm -f "$OUT/OriginalIntro.md"
[ -s "$OUT/review-history.json" ] || rm -f "$OUT/review-history.json"
# --- 3. Artifacts ----------------------------------------------------------
c_report=$(fetch_to "$REPORT_PATH" "$OUT/$REPORT_FILE")
c_sess=$( fetch_to "/api/$KIND/$ID/log-file?lines=$LINES" "$OUT/session.jsonl")
c_eval=$( fetch_to "/api/$KIND/$ID/eval-log-file?lines=$LINES" "$OUT/eval.jsonl")
c_evid=$( fetch_to "/api/$KIND/$ID/evidence" "$OUT/evidence.json")
c_ask=$( fetch_to "/api/$KIND/$ID/asking/history" "$OUT/asking.json")
c_read=$( fetch_to "/api/$KIND/$ID/readable-report" "$OUT/readable-report.json")
c_audit="-"
[ "$KIND" = "verifications" ] && c_audit=$(fetch_to "/api/verifications/$ID/audit-report" "$OUT/audit-report.json")
# --- 3b. Every round's session log + step timing, full-size ---------------
# log-file above is the latest round only, and capped at 8 MB by the API. Long
# runs need every round and the whole file, so: list logs/, fetch each, and when
# the API copy is shorter than the node says the file is, scp the original.
LOGS_N=0; LOGS_SSH=0; LOGS_SHORT=""
NODE=$(jq -r '.executorNodeId // empty' "$OUT/meta.json"); NODE_ALIAS="${NODE%%.*}"
WS=$(jq -r '.workspacePath // empty' "$OUT/meta.json")
if fetch_to "/api/$KIND/$ID/logs" "$OUT/logs.json" >/dev/null && [ -s "$OUT/logs.json" ]; then
mkdir -p "$OUT/logs"
while IFS=$'\t' read -r name size; do
[ -n "$name" ] || continue
fetch_to "/api/$KIND/$ID/logs/$name?lines=$LINES" "$OUT/logs/$name" >/dev/null
LOGS_N=$((LOGS_N + 1))
got=$(wc -c < "$OUT/logs/$name" 2>/dev/null || echo 0)
if [ "${size%.*}" -gt $((got + 1024)) ]; then
if [ "$NO_SSH" = "0" ] && [ -n "$NODE_ALIAS" ] && [ -n "$WS" ] \
&& scp -q -o BatchMode=yes -o ConnectTimeout=8 "$NODE_ALIAS:$WS/logs/$name" "$OUT/logs/$name" 2>/dev/null; then
LOGS_SSH=$((LOGS_SSH + 1))
else
LOGS_SHORT="$LOGS_SHORT $name"
fi
fi
done < <(jq -r '.files[]? | select(.kind == "session" or .kind == "timing" or .kind == "runner" or .kind == "critic") | [.name, (.size|tostring)] | @tsv' "$OUT/logs.json" 2>/dev/null)
fi
# readable report is JSON-wrapped HTML — unwrap so it can be read/opened directly.
if [ -s "$OUT/readable-report.json" ]; then
jq -r '.html // empty' "$OUT/readable-report.json" > "$OUT/readable-report.html" 2>/dev/null
[ -s "$OUT/readable-report.html" ] || rm -f "$OUT/readable-report.html"
fi
# --- 4. Optional: pull evidence images so they can be looked at ------------
EVID_N=0
if [ "$WITH_EVIDENCE" = "1" ] && [ -s "$OUT/evidence.json" ]; then
mkdir -p "$OUT/evidence"
while IFS=$'\t' read -r name url; do
[ -n "$url" ] || continue
curl -sS -m 120 -o "$OUT/evidence/$name" "$url" 2>/dev/null && EVID_N=$((EVID_N + 1))
done < <(jq -r '.files[]? | select(.isImage == true) | [.name, .url] | @tsv' "$OUT/evidence.json" 2>/dev/null | head -40)
fi
# --- 5. Report what we got -------------------------------------------------
{
echo "kind : $KIND"
echo "id : $ID"
echo "title : $(jq -r '.title // "?"' "$OUT/meta.json")"
echo "status : $(jq -r '.status // "?"' "$OUT/meta.json") attempts=$(jq -r '.attempts // 0' "$OUT/meta.json") platform=$(jq -r '.platform // "?"' "$OUT/meta.json")"
echo "node : $(jq -r '.executorNodeId // "-"' "$OUT/meta.json")"
echo "branch/MR : $(jq -r '.branchName // "-"' "$OUT/meta.json") | $(jq -r '.mrUrl // "-"' "$OUT/meta.json")"
echo "failure : $(jq -r '(.failureReason // "-") | gsub("\n"; " ⏎ ") | .[0:400]' "$OUT/meta.json")"
echo
echo "artifacts (http codes; 404 = that artifact does not exist for this id):"
echo " $REPORT_FILE=$c_report session.jsonl=$c_sess eval.jsonl=$c_eval evidence.json=$c_evid asking.json=$c_ask readable=$c_read audit-report.json=$c_audit"
echo " logs/: $LOGS_N files (per round)$( [ "$LOGS_SSH" -gt 0 ] && echo ", $LOGS_SSH pulled full-size from $NODE_ALIAS via scp" )"
[ -n "$LOGS_SHORT" ] && echo " ⚠ still truncated at the API's 8 MB (node unreachable or --no-ssh):$LOGS_SHORT"
[ "$WITH_EVIDENCE" = "1" ] && echo " evidence images downloaded: $EVID_N"
echo
echo "OUT=$OUT"
echo "KIND=$KIND"
} | tee "$OUT/fetch-summary.txt"