Initial import of streamline
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* One-off 存量 backfill for the /lifemon ledger (`task_transitions`).
|
||||
*
|
||||
* The live chokepoint (db.ts:notifyStatusTransition → recordTransition) only
|
||||
* started capturing `source='forward'` rows on deploy day, so every task that
|
||||
* predates it is invisible in /lifemon. This reconstructs a COARSE skeleton for
|
||||
* each historical bug/feature/security from the sparse timestamps the row still
|
||||
* carries, and writes it with `source='backfill'`.
|
||||
*
|
||||
* FIDELITY (read before trusting the numbers):
|
||||
* Reliable → EXEC start (startedAt), rework-round boundaries (each HUMAN
|
||||
* reviewHistory note = a manual_review bounce → next EXEC round),
|
||||
* terminal (finishedAt / status).
|
||||
* Dropped → the createdAt→startedAt span. Its real EVAL compute is only
|
||||
* minutes, but it is dominated by pending_confirmation + queue
|
||||
* wait whose entry timestamps were never stored — labelling the
|
||||
* whole span EVAL inflated it to week-long tails (workShare→1), so
|
||||
* we emit NO pre-exec segment. Historical EVAL is therefore blank
|
||||
* in backfill; only forward capture yields trustworthy EVAL.
|
||||
* NOT stored → manual_review / info_required dwell has no entry timestamp and
|
||||
* is folded into the adjacent EXEC. So backfill still under-counts
|
||||
* human WAIT — exactly why lifecycle_db keeps it out of headline
|
||||
* percentiles and the Portal shows it only behind `含回填`.
|
||||
* `critic` notes fire inside `fixing` (not a status change) and
|
||||
* are ignored here.
|
||||
*
|
||||
* Idempotent (INSERT OR IGNORE on the derived id; re-runs never duplicate and
|
||||
* never clobber a colliding forward row). DRY-RUN by default — prints the
|
||||
* reconstructed shape + the exact matrix /lifemon will render. Pass --commit to
|
||||
* actually insert:
|
||||
* bun scripts/backfill_lifecycle.ts # preview only
|
||||
* bun scripts/backfill_lifecycle.ts --commit # write source='backfill'
|
||||
*/
|
||||
import { listTasks, listFeatures, listSecurity } from "../control-server/src/db";
|
||||
import {
|
||||
insertBackfillTransitions,
|
||||
computeLifecycleStats,
|
||||
segClassOf,
|
||||
type BackfillTx,
|
||||
type TxKind,
|
||||
type TransitionRow,
|
||||
} from "../control-server/src/lifecycle_db";
|
||||
|
||||
const COMMIT = process.argv.includes("--commit");
|
||||
|
||||
/** The EXEC status name per task kind (initializing/EVAL is shared). */
|
||||
const EXEC_STATUS: Record<TxKind, string> = {
|
||||
bug: "fixing",
|
||||
feature: "implementing",
|
||||
security: "security_running",
|
||||
};
|
||||
|
||||
/** A status is terminal iff it holds no dwell we care about (segClassOf). */
|
||||
const isTerminal = (s: string) => segClassOf(s) === "TERMINAL";
|
||||
|
||||
/** Minimal shape shared by all three kinds' rows for reconstruction. */
|
||||
interface Hist {
|
||||
id: string;
|
||||
status: string;
|
||||
createdAt: number | null;
|
||||
startedAt: number | null;
|
||||
finishedAt: number | null;
|
||||
statusChangedAt: number | null;
|
||||
attempts: number | null;
|
||||
executorNodeId: string | null;
|
||||
reviewHistory: { at: number; source: "human" | "critic"; round: number }[] | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconstruct one target's ordered transitions. Emits strictly-increasing-ts
|
||||
* events only (out-of-order timestamps dropped, matching segmentsFor's skew
|
||||
* guard); a non-terminal current status is left OPEN (no trailing transition)
|
||||
* so its dwell isn't fabricated — forward capture owns it from now on.
|
||||
*/
|
||||
function reconstruct(kind: TxKind, h: Hist): BackfillTx[] {
|
||||
if (h.startedAt == null) return []; // no EXEC anchor → nothing defensible to emit
|
||||
const exec = EXEC_STATUS[kind];
|
||||
const node = h.executorNodeId ?? null;
|
||||
|
||||
// 1) collect candidate (status, ts) points, then enforce monotonic ts.
|
||||
// The createdAt→startedAt span (EVAL + confirm/queue wait) is deliberately
|
||||
// NOT emitted — its wait phases have no stored entry ts (see header).
|
||||
const pts: { status: string; ts: number; round: number | null }[] = [];
|
||||
pts.push({ status: exec, ts: h.startedAt, round: 1 });
|
||||
// each HUMAN review note = a manual_review bounce → a fresh EXEC round starts.
|
||||
for (const n of (h.reviewHistory ?? []).filter((n) => n.source === "human")) {
|
||||
if (n.at > 0) pts.push({ status: exec, ts: n.at, round: (n.round ?? 0) + 1 });
|
||||
}
|
||||
// terminal cap — only if the row actually reached a terminal status.
|
||||
if (isTerminal(h.status)) {
|
||||
const termTs = h.finishedAt ?? h.statusChangedAt;
|
||||
if (termTs != null) pts.push({ status: h.status, ts: termTs, round: h.attempts ?? null });
|
||||
}
|
||||
|
||||
pts.sort((a, b) => a.ts - b.ts);
|
||||
// drop non-increasing ts (dedupe / skew), keeping the earliest at each ts.
|
||||
const clean: typeof pts = [];
|
||||
for (const p of pts) if (!clean.length || p.ts > clean[clean.length - 1].ts) clean.push(p);
|
||||
if (clean.length < 2) return []; // need ≥1 closed segment
|
||||
|
||||
const out: BackfillTx[] = [];
|
||||
let prev: string | null = null;
|
||||
for (const p of clean) {
|
||||
out.push({ kind, targetId: h.id, fromStatus: prev, toStatus: p.status, round: p.round, nodeId: node, ts: p.ts });
|
||||
prev = p.status;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collect(): BackfillTx[] {
|
||||
const rows: BackfillTx[] = [];
|
||||
const push = (kind: TxKind, items: Hist[]) => {
|
||||
let contributed = 0;
|
||||
for (const h of items) {
|
||||
const r = reconstruct(kind, h);
|
||||
if (r.length) { rows.push(...r); contributed++; }
|
||||
}
|
||||
console.log(` ${kind.padEnd(9)} ${items.length} targets → ${contributed} reconstructed, ${rows.filter((x) => x.kind === kind).length} transitions`);
|
||||
};
|
||||
push("bug", listTasks() as unknown as Hist[]);
|
||||
push("feature", listFeatures() as unknown as Hist[]);
|
||||
push("security", listSecurity() as unknown as Hist[]);
|
||||
return rows;
|
||||
}
|
||||
|
||||
console.log(`[backfill_lifecycle] mode=${COMMIT ? "COMMIT" : "DRY-RUN"}`);
|
||||
const backfill = collect();
|
||||
|
||||
// Preview the exact matrix /lifemon will render for the backfill bucket.
|
||||
const rowsForStats: TransitionRow[] = backfill.map((r) => ({
|
||||
id: `${r.kind}:${r.targetId}:${r.toStatus}:${r.ts}`,
|
||||
kind: r.kind, targetId: r.targetId, fromStatus: r.fromStatus, toStatus: r.toStatus,
|
||||
round: r.round, nodeId: r.nodeId, source: "backfill", ts: r.ts,
|
||||
}));
|
||||
const minTs = rowsForStats.reduce((m, r) => Math.min(m, r.ts), Number.MAX_SAFE_INTEGER);
|
||||
const stats = computeLifecycleStats(rowsForStats, Number.isFinite(minTs) ? minTs : 0, true);
|
||||
|
||||
console.log(`\n[backfill_lifecycle] reconstructed ${backfill.length} transitions over ${stats.totals.tasks} targets`);
|
||||
console.log(" per-kind work/wait (backfill is WORK-biased by construction):");
|
||||
for (const k of stats.byKind) {
|
||||
console.log(` ${k.kind.padEnd(9)} tasks=${k.tasks} work=${Math.round(k.workMs / 3600000)}h wait=${Math.round(k.waitMs / 3600000)}h workShare=${k.workShare} rework=${k.reworkRate}`);
|
||||
}
|
||||
console.log(" segment matrix (kind | seg | count | p50 min | p90 min | max min):");
|
||||
for (const s of stats.segMatrix) {
|
||||
console.log(` ${s.kind.padEnd(9)} ${s.segClass.padEnd(12)} n=${String(s.count).padEnd(4)} p50=${(s.p50Ms / 60000).toFixed(1)} p90=${(s.p90Ms / 60000).toFixed(1)} max=${(s.maxMs / 60000).toFixed(1)}`);
|
||||
}
|
||||
|
||||
if (!COMMIT) {
|
||||
console.log("\n[backfill_lifecycle] DRY-RUN — nothing written. Re-run with --commit to insert source='backfill'.");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const inserted = insertBackfillTransitions(backfill);
|
||||
console.log(`\n[backfill_lifecycle] COMMITTED — ${inserted} new backfill rows (${backfill.length - inserted} already present, idempotent skip).`);
|
||||
Reference in New Issue
Block a user