Initial import of streamline
This commit is contained in:
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* 给存量的可读性报告生成记录补上「这是第几批」——`readable_report_runs.round`。
|
||||
*
|
||||
* bun scripts/backfill_readable_run_round.ts # 预演,只读,默认
|
||||
* bun scripts/backfill_readable_run_round.ts --commit # 真正写入
|
||||
* bun scripts/backfill_readable_run_round.ts --max-gap-h 3 # 放宽认领窗口
|
||||
*
|
||||
* ## 为什么需要它,以及为什么它是一个脚本而不是一条 UPDATE
|
||||
*
|
||||
* 0018 之前的 run 行没有批次号,所以批次切换器对存量任务只能给出「原始 markdown」
|
||||
* 那一半:那一批的大白话正文明明还在库里,却认不出它属于哪一批。这条脚本把它认回来。
|
||||
*
|
||||
* 认的依据是**时间上的因果**,不是猜:一次生成总是先把当前批次的原始报告归档下来
|
||||
* (archiveFixReportRound / archiveImplReportRound),紧接着才拿去生成正文。所以
|
||||
* 一份正文对应的批次,就是「归档时间早于这次生成、且离它最近」的那一批。生产上实测
|
||||
* 这个间隔是分钟级(第 9 批 13:48 归档、13:50 出正文)。
|
||||
*
|
||||
* 因果链断掉的地方就不认:
|
||||
* - 找不到任何早于它的归档批次 → 跳过;
|
||||
* - 最近的那一批离它超过 `--max-gap-h`(默认 1 小时)→ 跳过。中间大概率隔着一次
|
||||
* 实验室重跑或批量刷新,认过去就等于给一份正文安一个它没描述过的批次号,而这
|
||||
* 正是整套东西要防的那种假事实。
|
||||
*
|
||||
* 阈值不是拍的。2026-08-20 在生产上按同一规则只读预演过 1332 条 done+html 的记录:
|
||||
* 认得回 705 条,归档→生成的间隔中位 1.7 分钟、p90 3.9 分钟,而 p99 是 160 分钟 ——
|
||||
* 分布本身就把「正常的一次生成」和「隔了很久的重跑」分成了两坨,1 小时落在中间的
|
||||
* 空档里。剩下认不回的:474 条的目标压根没有按批归档(归档机制之前的老任务),
|
||||
* 128 条早于该目标的第一份归档。
|
||||
*
|
||||
* 预演是默认且严格只读:它把每一条的推断结果和跳过原因打出来,人看过再决定要不要
|
||||
* `--commit`。写生产是操作者的决定,不是 agent 顺手做的事(见 CLAUDE.md)。
|
||||
*/
|
||||
|
||||
import pg from "pg";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const COMMIT = args.includes("--commit");
|
||||
const gapArgIdx = args.findIndex((a) => a === "--max-gap-h");
|
||||
const MAX_GAP_H = gapArgIdx >= 0 ? Number(args[gapArgIdx + 1]) : 1;
|
||||
if (!Number.isFinite(MAX_GAP_H) || MAX_GAP_H <= 0) {
|
||||
console.error("--max-gap-h 需要一个正数小时数");
|
||||
process.exit(2);
|
||||
}
|
||||
const MAX_GAP_MS = MAX_GAP_H * 3600_000;
|
||||
|
||||
const raw = process.env.DATABASE_URL;
|
||||
if (!raw) {
|
||||
console.error("DATABASE_URL 没设置(见 .env)");
|
||||
process.exit(2);
|
||||
}
|
||||
const u = new URL(raw);
|
||||
const pool = new pg.Pool({
|
||||
host: u.hostname.replace(/^\[|\]$/g, ""),
|
||||
port: u.port ? Number(u.port) : 5432,
|
||||
user: decodeURIComponent(u.username),
|
||||
password: decodeURIComponent(u.password),
|
||||
database: u.pathname.replace(/^\//, ""),
|
||||
connectionTimeoutMillis: 30_000,
|
||||
});
|
||||
|
||||
interface RunRow { id: string; targetType: string; targetId: string; at: number; lang: string | null }
|
||||
interface ArchiveRow { targetId: string; round: number; createdAt: number }
|
||||
|
||||
const runs: RunRow[] = (await pool.query(
|
||||
`select id, target_type, target_id, coalesce(finished_at, created_at) as at, lang
|
||||
from readable_report_runs
|
||||
where round is null and status = 'done' and html is not null
|
||||
order by target_type, target_id, at`,
|
||||
)).rows.map((r) => ({
|
||||
id: r.id, targetType: r.target_type, targetId: r.target_id, at: Number(r.at), lang: r.lang,
|
||||
}));
|
||||
|
||||
const archives = new Map<string, ArchiveRow[]>();
|
||||
const loadArchives = async (kind: "task" | "feature") => {
|
||||
const table = kind === "task" ? "fix_report_versions" : "impl_report_versions";
|
||||
const idCol = kind === "task" ? "task_id" : "feature_id";
|
||||
const rows = (await pool.query(
|
||||
`select ${idCol} as target_id, round, created_at from ${table} order by ${idCol}, round`,
|
||||
)).rows;
|
||||
for (const r of rows) {
|
||||
const key = `${kind}:${r.target_id}`;
|
||||
const list = archives.get(key) ?? [];
|
||||
list.push({ targetId: r.target_id, round: Number(r.round), createdAt: Number(r.created_at) });
|
||||
archives.set(key, list);
|
||||
}
|
||||
};
|
||||
await loadArchives("task");
|
||||
await loadArchives("feature");
|
||||
|
||||
/** 归档早于这次生成、且离它最近的那一批。 */
|
||||
function inferRound(run: RunRow): { round: number; gapMs: number } | { skip: string } {
|
||||
const list = archives.get(`${run.targetType}:${run.targetId}`);
|
||||
if (!list?.length) return { skip: "该目标没有任何按批归档的原始报告" };
|
||||
let best: ArchiveRow | null = null;
|
||||
for (const a of list) {
|
||||
if (a.createdAt > run.at) continue;
|
||||
if (!best || a.round > best.round) best = a;
|
||||
}
|
||||
if (!best) return { skip: "这次生成早于该目标的第一份归档" };
|
||||
const gapMs = run.at - best.createdAt;
|
||||
if (gapMs > MAX_GAP_MS) return { skip: `离最近一批归档 ${(gapMs / 3600_000).toFixed(1)}h,超过 ${MAX_GAP_H}h` };
|
||||
return { round: best.round, gapMs };
|
||||
}
|
||||
|
||||
const decided: { id: string; round: number; gapMs: number; targetType: string; targetId: string; lang: string | null }[] = [];
|
||||
const skipped = new Map<string, number>();
|
||||
for (const run of runs) {
|
||||
const got = inferRound(run);
|
||||
if ("skip" in got) {
|
||||
skipped.set(got.skip.replace(/[\d.]+h/g, "Nh"), (skipped.get(got.skip.replace(/[\d.]+h/g, "Nh")) ?? 0) + 1);
|
||||
continue;
|
||||
}
|
||||
decided.push({ id: run.id, round: got.round, gapMs: got.gapMs, targetType: run.targetType, targetId: run.targetId, lang: run.lang });
|
||||
}
|
||||
|
||||
console.log(`存量待补的生成记录:${runs.length} 条`);
|
||||
console.log(`可推断:${decided.length} 条`);
|
||||
for (const [why, n] of skipped) console.log(`跳过:${why} —— ${n} 条`);
|
||||
|
||||
const gaps = decided.map((d) => d.gapMs).sort((a, b) => a - b);
|
||||
if (gaps.length) {
|
||||
const pct = (p: number) => gaps[Math.min(gaps.length - 1, Math.floor(gaps.length * p))];
|
||||
console.log(
|
||||
`归档→生成的间隔:中位 ${(pct(0.5) / 60000).toFixed(1)} 分钟,`
|
||||
+ `p90 ${(pct(0.9) / 60000).toFixed(1)} 分钟,最大 ${(gaps[gaps.length - 1] / 60000).toFixed(1)} 分钟`,
|
||||
);
|
||||
}
|
||||
console.log("\n前 15 条推断结果:");
|
||||
console.table(decided.slice(0, 15).map((d) => ({
|
||||
target: `${d.targetType}:${d.targetId.slice(0, 8)}`,
|
||||
lang: d.lang ?? "(旧)",
|
||||
round: d.round,
|
||||
gapMin: Number((d.gapMs / 60000).toFixed(1)),
|
||||
})));
|
||||
|
||||
if (!COMMIT) {
|
||||
console.log(`\n预演结束,一个字都没写。确认无误后加 --commit 执行。`);
|
||||
await pool.end();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// 写入:一条事务,按 id 逐条 UPDATE。条件里再带一次 `round is null`,这样即便脚本
|
||||
// 与一次正常生成撞车,也绝不会把新写进去的真实批次号覆盖成推断值。
|
||||
const client = await pool.connect();
|
||||
let written = 0;
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
for (const d of decided) {
|
||||
const res = await client.query(
|
||||
`update readable_report_runs set round = $1 where id = $2 and round is null`,
|
||||
[d.round, d.id],
|
||||
);
|
||||
written += res.rowCount ?? 0;
|
||||
}
|
||||
await client.query("COMMIT");
|
||||
} catch (err) {
|
||||
await client.query("ROLLBACK");
|
||||
console.error("回填失败,已回滚:", (err as Error).message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
console.log(`\n已写入 ${written} 条。`);
|
||||
await pool.end();
|
||||
Reference in New Issue
Block a user