Files
streamline/scripts/backfill_lifecycle.ts
T
2026-11-16 17:26:44 -08:00

155 lines
7.3 KiB
TypeScript

/**
* One-off 存量 backfill for the /lifemon ledger (`task_transitions`).
*
* The live chokepoint (db.ts:notifyStatusTransition → recordTransition) only
* started capturing `source='forward'` rows on deploy day, so every task that
* predates it is invisible in /lifemon. This reconstructs a COARSE skeleton for
* each historical bug/feature/security from the sparse timestamps the row still
* carries, and writes it with `source='backfill'`.
*
* FIDELITY (read before trusting the numbers):
* Reliable → EXEC start (startedAt), rework-round boundaries (each HUMAN
* reviewHistory note = a manual_review bounce → next EXEC round),
* terminal (finishedAt / status).
* Dropped → the createdAt→startedAt span. Its real EVAL compute is only
* minutes, but it is dominated by pending_confirmation + queue
* wait whose entry timestamps were never stored — labelling the
* whole span EVAL inflated it to week-long tails (workShare→1), so
* we emit NO pre-exec segment. Historical EVAL is therefore blank
* in backfill; only forward capture yields trustworthy EVAL.
* NOT stored → manual_review / info_required dwell has no entry timestamp and
* is folded into the adjacent EXEC. So backfill still under-counts
* human WAIT — exactly why lifecycle_db keeps it out of headline
* percentiles and the Portal shows it only behind `含回填`.
* `critic` notes fire inside `fixing` (not a status change) and
* are ignored here.
*
* Idempotent (INSERT OR IGNORE on the derived id; re-runs never duplicate and
* never clobber a colliding forward row). DRY-RUN by default — prints the
* reconstructed shape + the exact matrix /lifemon will render. Pass --commit to
* actually insert:
* bun scripts/backfill_lifecycle.ts # preview only
* bun scripts/backfill_lifecycle.ts --commit # write source='backfill'
*/
import { listTasks, listFeatures, listSecurity } from "../control-server/src/db";
import {
insertBackfillTransitions,
computeLifecycleStats,
segClassOf,
type BackfillTx,
type TxKind,
type TransitionRow,
} from "../control-server/src/lifecycle_db";
const COMMIT = process.argv.includes("--commit");
/** The EXEC status name per task kind (initializing/EVAL is shared). */
const EXEC_STATUS: Record<TxKind, string> = {
bug: "fixing",
feature: "implementing",
security: "security_running",
};
/** A status is terminal iff it holds no dwell we care about (segClassOf). */
const isTerminal = (s: string) => segClassOf(s) === "TERMINAL";
/** Minimal shape shared by all three kinds' rows for reconstruction. */
interface Hist {
id: string;
status: string;
createdAt: number | null;
startedAt: number | null;
finishedAt: number | null;
statusChangedAt: number | null;
attempts: number | null;
executorNodeId: string | null;
reviewHistory: { at: number; source: "human" | "critic"; round: number }[] | null;
}
/**
* Reconstruct one target's ordered transitions. Emits strictly-increasing-ts
* events only (out-of-order timestamps dropped, matching segmentsFor's skew
* guard); a non-terminal current status is left OPEN (no trailing transition)
* so its dwell isn't fabricated — forward capture owns it from now on.
*/
function reconstruct(kind: TxKind, h: Hist): BackfillTx[] {
if (h.startedAt == null) return []; // no EXEC anchor → nothing defensible to emit
const exec = EXEC_STATUS[kind];
const node = h.executorNodeId ?? null;
// 1) collect candidate (status, ts) points, then enforce monotonic ts.
// The createdAt→startedAt span (EVAL + confirm/queue wait) is deliberately
// NOT emitted — its wait phases have no stored entry ts (see header).
const pts: { status: string; ts: number; round: number | null }[] = [];
pts.push({ status: exec, ts: h.startedAt, round: 1 });
// each HUMAN review note = a manual_review bounce → a fresh EXEC round starts.
for (const n of (h.reviewHistory ?? []).filter((n) => n.source === "human")) {
if (n.at > 0) pts.push({ status: exec, ts: n.at, round: (n.round ?? 0) + 1 });
}
// terminal cap — only if the row actually reached a terminal status.
if (isTerminal(h.status)) {
const termTs = h.finishedAt ?? h.statusChangedAt;
if (termTs != null) pts.push({ status: h.status, ts: termTs, round: h.attempts ?? null });
}
pts.sort((a, b) => a.ts - b.ts);
// drop non-increasing ts (dedupe / skew), keeping the earliest at each ts.
const clean: typeof pts = [];
for (const p of pts) if (!clean.length || p.ts > clean[clean.length - 1].ts) clean.push(p);
if (clean.length < 2) return []; // need ≥1 closed segment
const out: BackfillTx[] = [];
let prev: string | null = null;
for (const p of clean) {
out.push({ kind, targetId: h.id, fromStatus: prev, toStatus: p.status, round: p.round, nodeId: node, ts: p.ts });
prev = p.status;
}
return out;
}
function collect(): BackfillTx[] {
const rows: BackfillTx[] = [];
const push = (kind: TxKind, items: Hist[]) => {
let contributed = 0;
for (const h of items) {
const r = reconstruct(kind, h);
if (r.length) { rows.push(...r); contributed++; }
}
console.log(` ${kind.padEnd(9)} ${items.length} targets → ${contributed} reconstructed, ${rows.filter((x) => x.kind === kind).length} transitions`);
};
push("bug", listTasks() as unknown as Hist[]);
push("feature", listFeatures() as unknown as Hist[]);
push("security", listSecurity() as unknown as Hist[]);
return rows;
}
console.log(`[backfill_lifecycle] mode=${COMMIT ? "COMMIT" : "DRY-RUN"}`);
const backfill = collect();
// Preview the exact matrix /lifemon will render for the backfill bucket.
const rowsForStats: TransitionRow[] = backfill.map((r) => ({
id: `${r.kind}:${r.targetId}:${r.toStatus}:${r.ts}`,
kind: r.kind, targetId: r.targetId, fromStatus: r.fromStatus, toStatus: r.toStatus,
round: r.round, nodeId: r.nodeId, source: "backfill", ts: r.ts,
}));
const minTs = rowsForStats.reduce((m, r) => Math.min(m, r.ts), Number.MAX_SAFE_INTEGER);
const stats = computeLifecycleStats(rowsForStats, Number.isFinite(minTs) ? minTs : 0, true);
console.log(`\n[backfill_lifecycle] reconstructed ${backfill.length} transitions over ${stats.totals.tasks} targets`);
console.log(" per-kind work/wait (backfill is WORK-biased by construction):");
for (const k of stats.byKind) {
console.log(` ${k.kind.padEnd(9)} tasks=${k.tasks} work=${Math.round(k.workMs / 3600000)}h wait=${Math.round(k.waitMs / 3600000)}h workShare=${k.workShare} rework=${k.reworkRate}`);
}
console.log(" segment matrix (kind | seg | count | p50 min | p90 min | max min):");
for (const s of stats.segMatrix) {
console.log(` ${s.kind.padEnd(9)} ${s.segClass.padEnd(12)} n=${String(s.count).padEnd(4)} p50=${(s.p50Ms / 60000).toFixed(1)} p90=${(s.p90Ms / 60000).toFixed(1)} max=${(s.maxMs / 60000).toFixed(1)}`);
}
if (!COMMIT) {
console.log("\n[backfill_lifecycle] DRY-RUN — nothing written. Re-run with --commit to insert source='backfill'.");
process.exit(0);
}
const inserted = insertBackfillTransitions(backfill);
console.log(`\n[backfill_lifecycle] COMMITTED — ${inserted} new backfill rows (${backfill.length - inserted} already present, idempotent skip).`);