134 lines
4.9 KiB
Bash
134 lines
4.9 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# Reap orphaned readable-report claude processes on a satellite node.
|
|
#
|
|
# ## Why this exists
|
|
#
|
|
# `bin/claude-pty.ts::killTree()` used to send SIGTERM and call process.exit() on
|
|
# the next line, never checking whether the tree died. A claude under `script`
|
|
# that was slow to handle the signal simply outlived its wrapper. On a node the
|
|
# control plane was retrying every few minutes (see the storm documented on
|
|
# readable_retry.ts::MAX_CONSECUTIVE_FAILURES) the leftovers accumulated until the
|
|
# machine could no longer start claude inside the wrapper's 120s transcript wait —
|
|
# at which point EVERY generation on it failed, producing more retries. Measured
|
|
# on temp-ios-01, 2026-08-22: 100+ claude processes.
|
|
#
|
|
# The wrapper is fixed, but a node that already has a pile needs the pile removed,
|
|
# and a node running an OLD bundle still produces them.
|
|
#
|
|
# ## Why it is safe to run on a live node
|
|
#
|
|
# Two independent guards, both necessary:
|
|
#
|
|
# 1. It matches only the first line of the two report prompts, which is in
|
|
# claude's own argv. A fix run, an evaluator run and the asking session carry
|
|
# completely different prompts and can never match. This is the difference
|
|
# between this script and `pkill -f claude`, which on a node with
|
|
# fixInFlight=1 kills a production task mid-implementation.
|
|
# 2. It ignores anything younger than MIN_AGE_MIN (default 15 min). A real
|
|
# generation is bounded by the satellite's own 5-minute claude timeout, and
|
|
# the control plane gives up at 5.5 minutes, so nothing older than that can
|
|
# still have a caller waiting for it. Below the threshold we might be looking
|
|
# at live work; above it, by construction, we are not.
|
|
#
|
|
# ## Usage
|
|
#
|
|
# # from a workstation — inspect first, it is read-only without --kill
|
|
# ssh <user>@<node> 'bash -s' < scripts/reap-orphan-report-agents.sh
|
|
# ssh <user>@<node> 'bash -s -- --kill' < scripts/reap-orphan-report-agents.sh
|
|
#
|
|
# MIN_AGE_MIN=30 ... raise the age floor (more conservative)
|
|
#
|
|
set -uo pipefail
|
|
|
|
DO_KILL=0
|
|
[ "${1:-}" = "--kill" ] && DO_KILL=1
|
|
MIN_AGE_MIN="${MIN_AGE_MIN:-15}"
|
|
MIN_AGE_S=$(( MIN_AGE_MIN * 60 ))
|
|
|
|
# First line of DEFAULT_READABLE_PROMPT_TEMPLATE (READABLE_PROMPT_TEMPLATE_MEDIA_HERO
|
|
# is derived from it by .replace() and keeps this line) and of the translate prompt.
|
|
# See shared/src/readable_prompt.ts.
|
|
PATTERNS=(
|
|
"You transform verbose engineering Markdown"
|
|
"You are translating an already-published HTML report fragment"
|
|
)
|
|
|
|
# ps etime is [[dd-]hh:]mm:ss — turn it into seconds.
|
|
etime_seconds() {
|
|
echo "$1" | awk -F: '
|
|
{ d=0; h=0;
|
|
if ($1 ~ /-/) { split($1, a, "-"); d=a[1]; $1=a[2] }
|
|
if (NF == 3) { h=$1; m=$2; s=$3 } else { m=$1; s=$2 }
|
|
print d*86400 + h*3600 + m*60 + s }'
|
|
}
|
|
|
|
echo "== node: $(hostname -s) | $(date '+%F %T')"
|
|
echo "== load: $(uptime | sed 's/.*load averages*: //')"
|
|
echo "== 年龄闸: 只处理运行超过 ${MIN_AGE_MIN} 分钟的报告渲染进程"
|
|
echo
|
|
|
|
victims=()
|
|
sessions=()
|
|
total=0
|
|
|
|
for pat in "${PATTERNS[@]}"; do
|
|
while read -r pid; do
|
|
[ -z "$pid" ] && continue
|
|
total=$(( total + 1 ))
|
|
et=$(ps -o etime= -p "$pid" 2>/dev/null | tr -d ' ')
|
|
[ -z "$et" ] && continue
|
|
secs=$(etime_seconds "$et")
|
|
cmd=$(ps -o command= -p "$pid" 2>/dev/null)
|
|
sid=$(echo "$cmd" | sed -n 's/.*--session-id \([0-9a-fA-F-]\{36\}\).*/\1/p')
|
|
if [ "$secs" -ge "$MIN_AGE_S" ]; then
|
|
printf ' ORPHAN pid=%-7s age=%-12s session=%s\n' "$pid" "$et" "${sid:-?}"
|
|
victims+=("$pid")
|
|
[ -n "$sid" ] && sessions+=("$sid")
|
|
else
|
|
printf ' live? pid=%-7s age=%-12s session=%s (未达年龄闸,跳过)\n' "$pid" "$et" "${sid:-?}"
|
|
fi
|
|
done < <(pgrep -f "$pat" 2>/dev/null)
|
|
done
|
|
|
|
echo
|
|
echo "== 报告渲染进程合计: ${total} 其中判定为残留: ${#victims[@]}"
|
|
|
|
if [ "${#victims[@]}" -eq 0 ]; then
|
|
echo "== 无需清理。"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$DO_KILL" -eq 0 ]; then
|
|
echo "== 这是只读预检。确认无误后加 --kill 再跑一次。"
|
|
exit 0
|
|
fi
|
|
|
|
echo
|
|
echo "== SIGTERM …"
|
|
for pid in "${victims[@]}"; do kill -TERM "$pid" 2>/dev/null; done
|
|
# The wrapper's tree (sh -c / tail -f / script) shares the session id in argv.
|
|
for sid in "${sessions[@]}"; do pkill -TERM -f "$sid" 2>/dev/null; done
|
|
sleep 5
|
|
|
|
echo "== 复查,对仍活着的升级到 SIGKILL …"
|
|
left=0
|
|
for pid in "${victims[@]}"; do
|
|
if kill -0 "$pid" 2>/dev/null; then kill -KILL "$pid" 2>/dev/null; left=$(( left + 1 )); fi
|
|
done
|
|
for sid in "${sessions[@]}"; do pkill -KILL -f "$sid" 2>/dev/null; done
|
|
sleep 2
|
|
echo "== 需要 SIGKILL 的: ${left}"
|
|
|
|
# Only the fifos belonging to sessions we just reaped — a live fix run has its own.
|
|
for sid in "${sessions[@]}"; do rm -f "/tmp/claude-pty-${sid}.in" 2>/dev/null; done
|
|
|
|
echo
|
|
remaining=0
|
|
for pat in "${PATTERNS[@]}"; do
|
|
n=$(pgrep -f "$pat" 2>/dev/null | wc -l | tr -d ' ')
|
|
remaining=$(( remaining + n ))
|
|
done
|
|
echo "== 清理后剩余报告渲染进程: ${remaining}"
|
|
echo "== load: $(uptime | sed 's/.*load averages*: //')"
|