#!/usr/bin/env bash # # Reap orphaned readable-report claude processes on a satellite node. # # ## Why this exists # # `bin/claude-pty.ts::killTree()` used to send SIGTERM and call process.exit() on # the next line, never checking whether the tree died. A claude under `script` # that was slow to handle the signal simply outlived its wrapper. On a node the # control plane was retrying every few minutes (see the storm documented on # readable_retry.ts::MAX_CONSECUTIVE_FAILURES) the leftovers accumulated until the # machine could no longer start claude inside the wrapper's 120s transcript wait — # at which point EVERY generation on it failed, producing more retries. Measured # on temp-ios-01, 2026-08-22: 100+ claude processes. # # The wrapper is fixed, but a node that already has a pile needs the pile removed, # and a node running an OLD bundle still produces them. # # ## Why it is safe to run on a live node # # Two independent guards, both necessary: # # 1. It matches only the first line of the two report prompts, which is in # claude's own argv. A fix run, an evaluator run and the asking session carry # completely different prompts and can never match. This is the difference # between this script and `pkill -f claude`, which on a node with # fixInFlight=1 kills a production task mid-implementation. # 2. It ignores anything younger than MIN_AGE_MIN (default 15 min). A real # generation is bounded by the satellite's own 5-minute claude timeout, and # the control plane gives up at 5.5 minutes, so nothing older than that can # still have a caller waiting for it. Below the threshold we might be looking # at live work; above it, by construction, we are not. # # ## Usage # # # from a workstation — inspect first, it is read-only without --kill # ssh @ 'bash -s' < scripts/reap-orphan-report-agents.sh # ssh @ 'bash -s -- --kill' < scripts/reap-orphan-report-agents.sh # # MIN_AGE_MIN=30 ... raise the age floor (more conservative) # set -uo pipefail DO_KILL=0 [ "${1:-}" = "--kill" ] && DO_KILL=1 MIN_AGE_MIN="${MIN_AGE_MIN:-15}" MIN_AGE_S=$(( MIN_AGE_MIN * 60 )) # First line of DEFAULT_READABLE_PROMPT_TEMPLATE (READABLE_PROMPT_TEMPLATE_MEDIA_HERO # is derived from it by .replace() and keeps this line) and of the translate prompt. # See shared/src/readable_prompt.ts. PATTERNS=( "You transform verbose engineering Markdown" "You are translating an already-published HTML report fragment" ) # ps etime is [[dd-]hh:]mm:ss — turn it into seconds. etime_seconds() { echo "$1" | awk -F: ' { d=0; h=0; if ($1 ~ /-/) { split($1, a, "-"); d=a[1]; $1=a[2] } if (NF == 3) { h=$1; m=$2; s=$3 } else { m=$1; s=$2 } print d*86400 + h*3600 + m*60 + s }' } echo "== node: $(hostname -s) | $(date '+%F %T')" echo "== load: $(uptime | sed 's/.*load averages*: //')" echo "== 年龄闸: 只处理运行超过 ${MIN_AGE_MIN} 分钟的报告渲染进程" echo victims=() sessions=() total=0 for pat in "${PATTERNS[@]}"; do while read -r pid; do [ -z "$pid" ] && continue total=$(( total + 1 )) et=$(ps -o etime= -p "$pid" 2>/dev/null | tr -d ' ') [ -z "$et" ] && continue secs=$(etime_seconds "$et") cmd=$(ps -o command= -p "$pid" 2>/dev/null) sid=$(echo "$cmd" | sed -n 's/.*--session-id \([0-9a-fA-F-]\{36\}\).*/\1/p') if [ "$secs" -ge "$MIN_AGE_S" ]; then printf ' ORPHAN pid=%-7s age=%-12s session=%s\n' "$pid" "$et" "${sid:-?}" victims+=("$pid") [ -n "$sid" ] && sessions+=("$sid") else printf ' live? pid=%-7s age=%-12s session=%s (未达年龄闸,跳过)\n' "$pid" "$et" "${sid:-?}" fi done < <(pgrep -f "$pat" 2>/dev/null) done echo echo "== 报告渲染进程合计: ${total} 其中判定为残留: ${#victims[@]}" if [ "${#victims[@]}" -eq 0 ]; then echo "== 无需清理。" exit 0 fi if [ "$DO_KILL" -eq 0 ]; then echo "== 这是只读预检。确认无误后加 --kill 再跑一次。" exit 0 fi echo echo "== SIGTERM …" for pid in "${victims[@]}"; do kill -TERM "$pid" 2>/dev/null; done # The wrapper's tree (sh -c / tail -f / script) shares the session id in argv. for sid in "${sessions[@]}"; do pkill -TERM -f "$sid" 2>/dev/null; done sleep 5 echo "== 复查,对仍活着的升级到 SIGKILL …" left=0 for pid in "${victims[@]}"; do if kill -0 "$pid" 2>/dev/null; then kill -KILL "$pid" 2>/dev/null; left=$(( left + 1 )); fi done for sid in "${sessions[@]}"; do pkill -KILL -f "$sid" 2>/dev/null; done sleep 2 echo "== 需要 SIGKILL 的: ${left}" # Only the fifos belonging to sessions we just reaped — a live fix run has its own. for sid in "${sessions[@]}"; do rm -f "/tmp/claude-pty-${sid}.in" 2>/dev/null; done echo remaining=0 for pat in "${PATTERNS[@]}"; do n=$(pgrep -f "$pat" 2>/dev/null | wc -l | tr -d ' ') remaining=$(( remaining + n )) done echo "== 清理后剩余报告渲染进程: ${remaining}" echo "== load: $(uptime | sed 's/.*load averages*: //')"