Files
streamline/scripts/reap-orphan-report-agents.sh
2026-11-16 17:26:44 -08:00

134 lines
4.9 KiB
Bash

#!/usr/bin/env bash
#
# Reap orphaned readable-report claude processes on a satellite node.
#
# ## Why this exists
#
# `bin/claude-pty.ts::killTree()` used to send SIGTERM and call process.exit() on
# the next line, never checking whether the tree died. A claude under `script`
# that was slow to handle the signal simply outlived its wrapper. On a node the
# control plane was retrying every few minutes (see the storm documented on
# readable_retry.ts::MAX_CONSECUTIVE_FAILURES) the leftovers accumulated until the
# machine could no longer start claude inside the wrapper's 120s transcript wait —
# at which point EVERY generation on it failed, producing more retries. Measured
# on temp-ios-01, 2026-08-22: 100+ claude processes.
#
# The wrapper is fixed, but a node that already has a pile needs the pile removed,
# and a node running an OLD bundle still produces them.
#
# ## Why it is safe to run on a live node
#
# Two independent guards, both necessary:
#
# 1. It matches only the first line of the two report prompts, which is in
# claude's own argv. A fix run, an evaluator run and the asking session carry
# completely different prompts and can never match. This is the difference
# between this script and `pkill -f claude`, which on a node with
# fixInFlight=1 kills a production task mid-implementation.
# 2. It ignores anything younger than MIN_AGE_MIN (default 15 min). A real
# generation is bounded by the satellite's own 5-minute claude timeout, and
# the control plane gives up at 5.5 minutes, so nothing older than that can
# still have a caller waiting for it. Below the threshold we might be looking
# at live work; above it, by construction, we are not.
#
# ## Usage
#
# # from a workstation — inspect first, it is read-only without --kill
# ssh <user>@<node> 'bash -s' < scripts/reap-orphan-report-agents.sh
# ssh <user>@<node> 'bash -s -- --kill' < scripts/reap-orphan-report-agents.sh
#
# MIN_AGE_MIN=30 ... raise the age floor (more conservative)
#
set -uo pipefail
DO_KILL=0
[ "${1:-}" = "--kill" ] && DO_KILL=1
MIN_AGE_MIN="${MIN_AGE_MIN:-15}"
MIN_AGE_S=$(( MIN_AGE_MIN * 60 ))
# First line of DEFAULT_READABLE_PROMPT_TEMPLATE (READABLE_PROMPT_TEMPLATE_MEDIA_HERO
# is derived from it by .replace() and keeps this line) and of the translate prompt.
# See shared/src/readable_prompt.ts.
PATTERNS=(
"You transform verbose engineering Markdown"
"You are translating an already-published HTML report fragment"
)
# ps etime is [[dd-]hh:]mm:ss — turn it into seconds.
etime_seconds() {
echo "$1" | awk -F: '
{ d=0; h=0;
if ($1 ~ /-/) { split($1, a, "-"); d=a[1]; $1=a[2] }
if (NF == 3) { h=$1; m=$2; s=$3 } else { m=$1; s=$2 }
print d*86400 + h*3600 + m*60 + s }'
}
echo "== node: $(hostname -s) | $(date '+%F %T')"
echo "== load: $(uptime | sed 's/.*load averages*: //')"
echo "== 年龄闸: 只处理运行超过 ${MIN_AGE_MIN} 分钟的报告渲染进程"
echo
victims=()
sessions=()
total=0
for pat in "${PATTERNS[@]}"; do
while read -r pid; do
[ -z "$pid" ] && continue
total=$(( total + 1 ))
et=$(ps -o etime= -p "$pid" 2>/dev/null | tr -d ' ')
[ -z "$et" ] && continue
secs=$(etime_seconds "$et")
cmd=$(ps -o command= -p "$pid" 2>/dev/null)
sid=$(echo "$cmd" | sed -n 's/.*--session-id \([0-9a-fA-F-]\{36\}\).*/\1/p')
if [ "$secs" -ge "$MIN_AGE_S" ]; then
printf ' ORPHAN pid=%-7s age=%-12s session=%s\n' "$pid" "$et" "${sid:-?}"
victims+=("$pid")
[ -n "$sid" ] && sessions+=("$sid")
else
printf ' live? pid=%-7s age=%-12s session=%s (未达年龄闸,跳过)\n' "$pid" "$et" "${sid:-?}"
fi
done < <(pgrep -f "$pat" 2>/dev/null)
done
echo
echo "== 报告渲染进程合计: ${total} 其中判定为残留: ${#victims[@]}"
if [ "${#victims[@]}" -eq 0 ]; then
echo "== 无需清理。"
exit 0
fi
if [ "$DO_KILL" -eq 0 ]; then
echo "== 这是只读预检。确认无误后加 --kill 再跑一次。"
exit 0
fi
echo
echo "== SIGTERM …"
for pid in "${victims[@]}"; do kill -TERM "$pid" 2>/dev/null; done
# The wrapper's tree (sh -c / tail -f / script) shares the session id in argv.
for sid in "${sessions[@]}"; do pkill -TERM -f "$sid" 2>/dev/null; done
sleep 5
echo "== 复查,对仍活着的升级到 SIGKILL …"
left=0
for pid in "${victims[@]}"; do
if kill -0 "$pid" 2>/dev/null; then kill -KILL "$pid" 2>/dev/null; left=$(( left + 1 )); fi
done
for sid in "${sessions[@]}"; do pkill -KILL -f "$sid" 2>/dev/null; done
sleep 2
echo "== 需要 SIGKILL 的: ${left}"
# Only the fifos belonging to sessions we just reaped — a live fix run has its own.
for sid in "${sessions[@]}"; do rm -f "/tmp/claude-pty-${sid}.in" 2>/dev/null; done
echo
remaining=0
for pat in "${PATTERNS[@]}"; do
n=$(pgrep -f "$pat" 2>/dev/null | wc -l | tr -d ' ')
remaining=$(( remaining + n ))
done
echo "== 清理后剩余报告渲染进程: ${remaining}"
echo "== load: $(uptime | sed 's/.*load averages*: //')"