Initial import of streamline

This commit is contained in:
2026-11-16 17:26:44 -08:00
commit 8e4aec1855
1221 changed files with 274256 additions and 0 deletions
+164
View File
@@ -0,0 +1,164 @@
/**
* 给存量的可读性报告生成记录补上「这是第几批」——`readable_report_runs.round`。
*
* bun scripts/backfill_readable_run_round.ts # 预演,只读,默认
* bun scripts/backfill_readable_run_round.ts --commit # 真正写入
* bun scripts/backfill_readable_run_round.ts --max-gap-h 3 # 放宽认领窗口
*
* ## 为什么需要它,以及为什么它是一个脚本而不是一条 UPDATE
*
* 0018 之前的 run 行没有批次号,所以批次切换器对存量任务只能给出「原始 markdown」
* 那一半:那一批的大白话正文明明还在库里,却认不出它属于哪一批。这条脚本把它认回来。
*
* 认的依据是**时间上的因果**,不是猜:一次生成总是先把当前批次的原始报告归档下来
* (archiveFixReportRound / archiveImplReportRound),紧接着才拿去生成正文。所以
* 一份正文对应的批次,就是「归档时间早于这次生成、且离它最近」的那一批。生产上实测
* 这个间隔是分钟级(第 9 批 13:48 归档、13:50 出正文)。
*
* 因果链断掉的地方就不认:
* - 找不到任何早于它的归档批次 → 跳过;
* - 最近的那一批离它超过 `--max-gap-h`(默认 1 小时)→ 跳过。中间大概率隔着一次
* 实验室重跑或批量刷新,认过去就等于给一份正文安一个它没描述过的批次号,而这
* 正是整套东西要防的那种假事实。
*
* 阈值不是拍的。2026-08-20 在生产上按同一规则只读预演过 1332 条 done+html 的记录:
* 认得回 705 条,归档→生成的间隔中位 1.7 分钟、p90 3.9 分钟,而 p99 是 160 分钟 ——
* 分布本身就把「正常的一次生成」和「隔了很久的重跑」分成了两坨,1 小时落在中间的
* 空档里。剩下认不回的:474 条的目标压根没有按批归档(归档机制之前的老任务),
* 128 条早于该目标的第一份归档。
*
* 预演是默认且严格只读:它把每一条的推断结果和跳过原因打出来,人看过再决定要不要
* `--commit`。写生产是操作者的决定,不是 agent 顺手做的事(见 CLAUDE.md)。
*/
import pg from "pg";
const args = process.argv.slice(2);
const COMMIT = args.includes("--commit");
const gapArgIdx = args.findIndex((a) => a === "--max-gap-h");
const MAX_GAP_H = gapArgIdx >= 0 ? Number(args[gapArgIdx + 1]) : 1;
if (!Number.isFinite(MAX_GAP_H) || MAX_GAP_H <= 0) {
console.error("--max-gap-h 需要一个正数小时数");
process.exit(2);
}
const MAX_GAP_MS = MAX_GAP_H * 3600_000;
const raw = process.env.DATABASE_URL;
if (!raw) {
console.error("DATABASE_URL 没设置(见 .env)");
process.exit(2);
}
const u = new URL(raw);
const pool = new pg.Pool({
host: u.hostname.replace(/^\[|\]$/g, ""),
port: u.port ? Number(u.port) : 5432,
user: decodeURIComponent(u.username),
password: decodeURIComponent(u.password),
database: u.pathname.replace(/^\//, ""),
connectionTimeoutMillis: 30_000,
});
interface RunRow { id: string; targetType: string; targetId: string; at: number; lang: string | null }
interface ArchiveRow { targetId: string; round: number; createdAt: number }
const runs: RunRow[] = (await pool.query(
`select id, target_type, target_id, coalesce(finished_at, created_at) as at, lang
from readable_report_runs
where round is null and status = 'done' and html is not null
order by target_type, target_id, at`,
)).rows.map((r) => ({
id: r.id, targetType: r.target_type, targetId: r.target_id, at: Number(r.at), lang: r.lang,
}));
const archives = new Map<string, ArchiveRow[]>();
const loadArchives = async (kind: "task" | "feature") => {
const table = kind === "task" ? "fix_report_versions" : "impl_report_versions";
const idCol = kind === "task" ? "task_id" : "feature_id";
const rows = (await pool.query(
`select ${idCol} as target_id, round, created_at from ${table} order by ${idCol}, round`,
)).rows;
for (const r of rows) {
const key = `${kind}:${r.target_id}`;
const list = archives.get(key) ?? [];
list.push({ targetId: r.target_id, round: Number(r.round), createdAt: Number(r.created_at) });
archives.set(key, list);
}
};
await loadArchives("task");
await loadArchives("feature");
/** 归档早于这次生成、且离它最近的那一批。 */
function inferRound(run: RunRow): { round: number; gapMs: number } | { skip: string } {
const list = archives.get(`${run.targetType}:${run.targetId}`);
if (!list?.length) return { skip: "该目标没有任何按批归档的原始报告" };
let best: ArchiveRow | null = null;
for (const a of list) {
if (a.createdAt > run.at) continue;
if (!best || a.round > best.round) best = a;
}
if (!best) return { skip: "这次生成早于该目标的第一份归档" };
const gapMs = run.at - best.createdAt;
if (gapMs > MAX_GAP_MS) return { skip: `离最近一批归档 ${(gapMs / 3600_000).toFixed(1)}h,超过 ${MAX_GAP_H}h` };
return { round: best.round, gapMs };
}
const decided: { id: string; round: number; gapMs: number; targetType: string; targetId: string; lang: string | null }[] = [];
const skipped = new Map<string, number>();
for (const run of runs) {
const got = inferRound(run);
if ("skip" in got) {
skipped.set(got.skip.replace(/[\d.]+h/g, "Nh"), (skipped.get(got.skip.replace(/[\d.]+h/g, "Nh")) ?? 0) + 1);
continue;
}
decided.push({ id: run.id, round: got.round, gapMs: got.gapMs, targetType: run.targetType, targetId: run.targetId, lang: run.lang });
}
console.log(`存量待补的生成记录:${runs.length} 条`);
console.log(`可推断:${decided.length} 条`);
for (const [why, n] of skipped) console.log(`跳过:${why} —— ${n} 条`);
const gaps = decided.map((d) => d.gapMs).sort((a, b) => a - b);
if (gaps.length) {
const pct = (p: number) => gaps[Math.min(gaps.length - 1, Math.floor(gaps.length * p))];
console.log(
`归档→生成的间隔:中位 ${(pct(0.5) / 60000).toFixed(1)} 分钟,`
+ `p90 ${(pct(0.9) / 60000).toFixed(1)} 分钟,最大 ${(gaps[gaps.length - 1] / 60000).toFixed(1)} 分钟`,
);
}
console.log("\n前 15 条推断结果:");
console.table(decided.slice(0, 15).map((d) => ({
target: `${d.targetType}:${d.targetId.slice(0, 8)}`,
lang: d.lang ?? "(旧)",
round: d.round,
gapMin: Number((d.gapMs / 60000).toFixed(1)),
})));
if (!COMMIT) {
console.log(`\n预演结束,一个字都没写。确认无误后加 --commit 执行。`);
await pool.end();
process.exit(0);
}
// 写入:一条事务,按 id 逐条 UPDATE。条件里再带一次 `round is null`,这样即便脚本
// 与一次正常生成撞车,也绝不会把新写进去的真实批次号覆盖成推断值。
const client = await pool.connect();
let written = 0;
try {
await client.query("BEGIN");
for (const d of decided) {
const res = await client.query(
`update readable_report_runs set round = $1 where id = $2 and round is null`,
[d.round, d.id],
);
written += res.rowCount ?? 0;
}
await client.query("COMMIT");
} catch (err) {
await client.query("ROLLBACK");
console.error("回填失败,已回滚:", (err as Error).message);
process.exit(1);
} finally {
client.release();
}
console.log(`\n已写入 ${written} 条。`);
await pool.end();