Files
streamline/scripts/reap-stuck-claude.sh
2026-11-16 17:26:44 -08:00

96 lines
4.1 KiB
Bash

#!/bin/bash
#
# reap-stuck-claude.sh — watchdog for the Streamline bug + feature runners.
#
# Symptom it fixes: a `claude -p` session finishes (prints its terminal
# `STATUS:` line and delivers the MR) but the OS process never exits because the
# agent left non-terminating background shells (e.g. `until pgrep …; do sleep; done`
# whose pattern self-matches, or `until adb …; do sleep; done` waiting on state
# that never arrives, or a stray `adb logcat` left running after verification).
# The runner blocks on `await proc.exited`, so the task is pinned in
# `fixing` / `implementing` forever.
#
# This reaper finds runner-spawned claude sessions whose log shows a terminal
# STATUS line AND has been idle for a while, then kills the whole process tree.
# Killing claude lets the server's runner resolve `proc.exited`, read the STATUS,
# and move the task forward (-> manual_review / fix_failed / unreproducible /
# implementation_failed). It then writes the agent's report — FixReport.md for
# bug runs, ImplReport.md for feature runs — to the workspace.
#
# Safe by design:
# - Only touches `claude -p` processes carrying `--remote-control streamline-`.
# - Requires the agent's own terminal STATUS line to be present (it declared
# completion) — never kills a session that hasn't finished.
# - Requires the log to be idle (no writes) for >= IDLE_MIN minutes, so an
# actively-working session (or one still streaming a build) is left alone.
#
# Env:
# IDLE_MIN minutes the session.log must be idle before reaping (default 15)
# REAP_DRY_RUN set to 1 to only log what WOULD be killed
#
# Schedule hourly via cron: 0 * * * * /Users/fei/BBBBBBug/scripts/reap-stuck-claude.sh
set -uo pipefail
IDLE_MIN="${IDLE_MIN:-15}"
DRY="${REAP_DRY_RUN:-0}"
LOG="${REAP_LOG:-$HOME/.streamline/reaper.log}"
mkdir -p "$(dirname "$LOG")"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG"; }
# Recursively kill a process and all its descendants (children first).
kill_tree() {
local pid=$1 child
for child in $(pgrep -P "$pid" 2>/dev/null); do
kill_tree "$child"
done
kill "$pid" 2>/dev/null
}
reaped=0
# All runner-spawned fix sessions.
for pid in $(pgrep -f -- '--remote-control streamline-' 2>/dev/null); do
cmd=$(ps -p "$pid" -o command= 2>/dev/null)
[ -z "$cmd" ] && continue
case "$cmd" in *"claude -p"*) ;; *) continue ;; esac
# Workspace dir from the prompt (…工作区在 /…/bug_workspace/<id>…).
# Bug runs are bug_workspace/<numericMeegoId>; feature runs are
# bug_workspace/feature-<uuid>. Charset is tight (digits or
# `feature-[a-z0-9-]+`) because the prompt has a Chinese 。 right after
# the path with no space — a permissive [^/ ]+ would swallow trailing
# literals like `。\012TikTok` and break the workspace path.
ws=$(printf '%s' "$cmd" | grep -oE '/[^ ]*bug_workspace/(feature-[a-z0-9-]+|[0-9]+)' | head -1)
[ -z "$ws" ] && { log "pid=$pid: no workspace in cmdline, skip"; continue; }
logfile="$ws/logs/session.log"
[ -f "$logfile" ] || { log "pid=$pid ws=$ws: no session.log yet, skip"; continue; }
# Has the agent declared completion? (its contract: final line is STATUS: …)
# Bug runner emits FIXED/UNREPRODUCIBLE/FIX_FAILED; feature runner emits
# IMPLEMENTED/IMPLEMENTATION_FAILED. Reap on any terminal token.
if ! grep -qE 'STATUS: (FIXED|UNREPRODUCIBLE|FIX_FAILED|IMPLEMENTED|IMPLEMENTATION_FAILED)' "$logfile"; then
continue
fi
# Is the log idle? (no writes for >= IDLE_MIN minutes => not actively working)
if [ -z "$(find "$logfile" -mmin +"$IDLE_MIN" 2>/dev/null)" ]; then
log "pid=$pid ws=$ws: STATUS present but log still fresh (<${IDLE_MIN}m), leaving"
continue
fi
status=$(grep -oE 'STATUS: (FIXED|UNREPRODUCIBLE|FIX_FAILED|IMPLEMENTED|IMPLEMENTATION_FAILED)' "$logfile" | tail -1)
kids=$(pgrep -P "$pid" 2>/dev/null | tr '\n' ' ')
if [ "$DRY" = "1" ]; then
log "[dry-run] WOULD reap pid=$pid ws=$ws ($status); descendants: ${kids:-none}"
else
log "reaping pid=$pid ws=$ws ($status); descendants: ${kids:-none}"
kill_tree "$pid"
reaped=$((reaped + 1))
fi
done
[ "$reaped" -gt 0 ] && log "reaped $reaped stuck session(s)"
exit 0