274 lines
14 KiB
TypeScript
274 lines
14 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
||
import { readFileSync } from "node:fs";
|
||
import path from "node:path";
|
||
import {
|
||
classifyJob, classifyZhongkuiIssues, extractZhongkuiIssues, jobResultDetails, judgeCi, newestCommit,
|
||
normalizeCiGateSettings, parseDynamicJobs, renderCiFixNote, summarizeCiGate, DEFAULT_CI_GATE_SETTINGS,
|
||
freshnessEligible, freshnessSourceFor, outOfDateOnHead, unsafeSyncSource, canFixNow, ciFollowStatuses, followedSince,
|
||
type CiGateRunView, type CiJob,
|
||
} from "@bbbug/shared/ci_gate";
|
||
import { computeRoundSummary, parseReviewHistory, renderReviewHistoryForPrompt } from "@bbbug/shared/types";
|
||
|
||
/**
|
||
* CI 收尾的纯逻辑。夹具是真实的 ezrecovery 回包(2026-10-14 抓取,裁掉了无关字段):
|
||
* - dynamic-jobs-8124837:iOS MR,check_t_zhongkui_pre 红(report 29215839,JoJo 导出编译指令失败)
|
||
* - dynamic-jobs-8176347:iOS MR,trigger_pipeline 报「分支落后超 5 天」,下游 job 都没铺出来
|
||
* - zhongkui-issues-31608096:钟馗 report,命中 TTKABTest 已弃用等代码规则
|
||
*/
|
||
const fx = (name: string) => JSON.parse(readFileSync(path.join(import.meta.dir, "fixtures/ci", name), "utf8"));
|
||
|
||
function job(over: Partial<CiJob>): CiJob {
|
||
return {
|
||
name: "x", stage: "check", status: "success", jobId: 1, commit: "a".repeat(40), url: "", errorMsg: "",
|
||
statusText: "", jobResult: "", allowFailed: false, created: 1, stopped: 2, ...over,
|
||
};
|
||
}
|
||
|
||
describe("解析与判真绿", () => {
|
||
test("真实回包:钟馗 pre 红", () => {
|
||
const jobs = parseDynamicJobs(fx("dynamic-jobs-8124837.json"))!;
|
||
const head = "9ecb48ac5af9210bde62512bd06be35dc411a298";
|
||
const j = judgeCi(jobs, head);
|
||
expect(j.settled).toBe(true);
|
||
expect(j.verdict).toBe("red");
|
||
expect(j.failed.map((f) => f.name)).toEqual(["check_t_zhongkui_pre"]);
|
||
// report 类不看明细分不了类
|
||
expect(classifyJob(j.failed[0]!)).toBeNull();
|
||
});
|
||
|
||
test("真实回包:分支落后", () => {
|
||
const jobs = parseDynamicJobs(fx("dynamic-jobs-8176347.json"))!;
|
||
const j = judgeCi(jobs, newestCommit(jobs));
|
||
expect(j.verdict).toBe("red");
|
||
expect(classifyJob(j.failed[0]!)).toBe("out_of_date");
|
||
});
|
||
|
||
test("还有 job 在跑 → 不下结论", () => {
|
||
const j = judgeCi([job({ status: "running" }), job({ status: "failed" })], "a".repeat(40));
|
||
expect(j.settled).toBe(false);
|
||
expect(j.verdict).toBeNull();
|
||
});
|
||
|
||
test("新 commit 一个 job 都还没铺出来 → 不能拿旧 commit 的结果下结论", () => {
|
||
const j = judgeCi([job({ commit: "b".repeat(40) })], "a".repeat(40), { quietForMs: 60 * 60_000 });
|
||
expect(j.settled).toBe(false);
|
||
});
|
||
|
||
test("旧 commit 的成功残留:宽限期内等,过了宽限期按当前 commit 判", () => {
|
||
const jobs = [job({ name: "build" }), job({ name: "live_lint", commit: "b".repeat(40) })];
|
||
expect(judgeCi(jobs, "a".repeat(40), { quietForMs: 0 }).settled).toBe(false);
|
||
const late = judgeCi(jobs, "a".repeat(40), { quietForMs: 20 * 60_000 });
|
||
expect(late.settled).toBe(true);
|
||
expect(late.verdict).toBe("green");
|
||
});
|
||
|
||
test("新流水线铺出一半:旧 commit 的失败不能当这次的结论", () => {
|
||
const jobs = [
|
||
job({ name: "trigger_pipeline" }),
|
||
job({ name: "check_t_zhongkui_pre", status: "failed", commit: "b".repeat(40) }),
|
||
];
|
||
expect(judgeCi(jobs, "a".repeat(40), { quietForMs: 60_000 }).settled).toBe(false);
|
||
// 过了宽限期还没被新流水线取代:Bits 上它就是红的
|
||
expect(judgeCi(jobs, "a".repeat(40), { quietForMs: 20 * 60_000 }).verdict).toBe("red");
|
||
});
|
||
|
||
test("同名 job 在新 commit 上跑过了,旧 commit 的失败作废", () => {
|
||
const jobs = [
|
||
job({ name: "check_t_zhongkui_pre" }),
|
||
job({ name: "check_t_zhongkui_pre", status: "failed", commit: "b".repeat(40) }),
|
||
];
|
||
const j = judgeCi(jobs, "a".repeat(40));
|
||
expect(j.settled).toBe(true);
|
||
expect(j.verdict).toBe("green");
|
||
});
|
||
|
||
test("allowFailed 的失败不卡合入,不算红", () => {
|
||
const j = judgeCi([job({}), job({ name: "perf", status: "failed", allowFailed: true })], "a".repeat(40));
|
||
expect(j.verdict).toBe("green");
|
||
});
|
||
|
||
test("skipped 不计", () => {
|
||
const j = judgeCi([job({}), job({ name: "build_m_base", status: "skipped" })], "a".repeat(40));
|
||
expect(j.verdict).toBe("green");
|
||
expect(j.total).toBe(1);
|
||
});
|
||
|
||
test("认不出的形状 → null", () => {
|
||
expect(parseDynamicJobs({ error: "x" })).toBeNull();
|
||
expect(parseDynamicJobs([])).toEqual([]);
|
||
});
|
||
});
|
||
|
||
describe("失败分类", () => {
|
||
test("超时 / 限流 / arrow 是基建", () => {
|
||
expect(classifyJob(job({ name: "build_m", status: "failed", errorMsg: "Task Execution Timeout" }))).toBe("infra");
|
||
expect(classifyJob(job({ name: "test_studio_unit", status: "failed", jobResult: "{\"step_error\":{\"error_message\":\"400 too many requests\"}}" }))).toBe("infra");
|
||
expect(classifyJob(job({ name: "test_notice_unit", status: "failed", jobResult: "arrow failed (exit=64)" }))).toBe("infra");
|
||
});
|
||
|
||
test("按 jobURL / 名字分", () => {
|
||
expect(classifyJob(job({ name: "z_check_code_check", url: "https://bits.bytedance.net/devops/1/code/detail/2/check?check_type=mt_static_code_analysis" }))).toBe("static");
|
||
expect(classifyJob(job({ name: "test_im_unit", jobResult: "{\"failed_fastlane_action\":\"xctest_upload_xcresult\"}" }))).toBe("unit_test");
|
||
expect(classifyJob(job({ name: "musically_android_build", errorMsg: "code:1 " }))).toBe("build");
|
||
expect(classifyJob(job({ name: "tiktok_live_lint_job", errorMsg: "code:1 " }))).toBe("rule");
|
||
expect(classifyJob(job({ name: "performance_gate", errorMsg: "有任何疑问,可联系 余剑 解决" }))).toBe("other");
|
||
});
|
||
|
||
test("钟馗:JoJo 导出失败是基建,TTKABTest 是代码规则", () => {
|
||
const infra = extractZhongkuiIssues(fx("zhongkui-issues-29215839.json"));
|
||
expect(infra.map((i) => i.rule)).toEqual(["JojoIssue"]);
|
||
expect(classifyZhongkuiIssues(infra)).toBe("infra");
|
||
const code = extractZhongkuiIssues(fx("zhongkui-issues-31608096.json"));
|
||
expect(code.some((i) => i.rule === "LimitKeywordsTTKABTest")).toBe(true);
|
||
expect(code.find((i) => i.rule === "LimitKeywordsTTKABTest")!.file).toBeTruthy();
|
||
expect(classifyZhongkuiIssues(code)).toBe("rule");
|
||
});
|
||
|
||
test("钟馗:只剩要审批的规则 → 交给人", () => {
|
||
expect(classifyZhongkuiIssues([{ rule: "LaunchCodeProtectionChecker", file: "a.m", line: 1, message: "加 FF 并审批" }])).toBe("approval");
|
||
expect(classifyZhongkuiIssues([
|
||
{ rule: "LaunchCodeProtectionChecker", file: "a.m", line: 1, message: "" },
|
||
{ rule: "ForbidUILabel", file: "a.m", line: 2, message: "" },
|
||
])).toBe("rule");
|
||
});
|
||
|
||
test("iOS jobResult 里的编译错带 file:line", () => {
|
||
const items = jobResultDetails(job({
|
||
jobResult: JSON.stringify({ build_errors: [{ file_path: "Mod/Tests/Foo.m:15:17", reason: "auto property synthesis will not synthesize" }] }),
|
||
}));
|
||
expect(items).toEqual([{ file: "Mod/Tests/Foo.m", line: 15, message: "auto property synthesis will not synthesize" }]);
|
||
});
|
||
});
|
||
|
||
describe("回炉意见", () => {
|
||
test("写明只修 CI、带 file:line 和链接", () => {
|
||
const text = renderCiFixNote([{
|
||
job: "check_t_zhongkui_pre", stage: "check", jobId: 1, url: "https://bits/x/report/1", category: "rule",
|
||
items: [{ file: "A.m", line: 3, rule: "ForbidUILabel", message: "请改用 TUXLabel" }],
|
||
}], { mrUrl: "https://bits/mr/1", commit: "abcdef1234567890" });
|
||
expect(text).toContain("CI 收尾自动回炉");
|
||
expect(text).toContain("不要改变已有修复的行为逻辑");
|
||
expect(text).toContain("A.m:3 ForbidUILabel — 请改用 TUXLabel");
|
||
expect(text).toContain("abcdef123456");
|
||
});
|
||
|
||
test("ci 来源的意见:进 prompt、不算人工打回", () => {
|
||
const history = JSON.stringify([
|
||
{ at: 1, source: "human", round: 1, text: "改一下" },
|
||
{ at: 2, source: "ci", round: 2, text: "CI 红了" },
|
||
]);
|
||
expect(parseReviewHistory(history).map((n) => n.source)).toEqual(["human", "ci"]);
|
||
expect(renderReviewHistoryForPrompt(history)).toContain("CI 收尾");
|
||
const s = computeRoundSummary({ attempts: 3, reviewHistory: history });
|
||
expect(s.humanRejects).toBe(1);
|
||
expect(s.ciRejects).toBe(1);
|
||
expect(s.unreviewedRuns).toBe(0);
|
||
});
|
||
});
|
||
|
||
describe("保鲜的门槛", () => {
|
||
const DAY = 24 * 60 * 60_000;
|
||
test("MR 开了 20 天以上不再保鲜;开单时间未知也不做", () => {
|
||
const now = 100 * DAY;
|
||
expect(freshnessEligible(now - 3 * DAY, now)).toBe(true);
|
||
expect(freshnessEligible(now - 20 * DAY, now)).toBe(true);
|
||
expect(freshnessEligible(now - 21 * DAY, now)).toBe(false);
|
||
expect(freshnessEligible(null, now)).toBe(false);
|
||
});
|
||
test("合并源固定 develop;设了别的基线的单不保鲜", () => {
|
||
expect(freshnessSourceFor(null)).toBe("develop");
|
||
expect(freshnessSourceFor("develop")).toBe("develop");
|
||
expect(freshnessSourceFor("feature/story-2026q4")).toBeNull();
|
||
});
|
||
test("只认 MR 最新 commit 上的「分支落后」", () => {
|
||
const jobs = parseDynamicJobs(fx("dynamic-jobs-8176347.json"))!;
|
||
const head = jobs[0]!.commit;
|
||
expect(outOfDateOnHead(jobs, head)).toBe(true);
|
||
// 保鲜推了新 commit 之后,旧 commit 那条还挂着:不再触发
|
||
expect(outOfDateOnHead(jobs, "c".repeat(40))).toBe(false);
|
||
expect(outOfDateOnHead(jobs, null)).toBe(false);
|
||
});
|
||
test("暂存分支不当合并源", () => {
|
||
expect(unsafeSyncSource("latest_but_unstable/dont_merge_unless_necessary")).toBe(true);
|
||
expect(unsafeSyncSource("develop")).toBe(false);
|
||
});
|
||
});
|
||
|
||
describe("设置", () => {
|
||
test("缺省 + 夹紧;旧版字段丢掉", () => {
|
||
expect(normalizeCiGateSettings(null)).toEqual(DEFAULT_CI_GATE_SETTINGS);
|
||
const s = normalizeCiGateSettings({ maxFixRounds: 99, followDays: 0, autoRetry: false, bugBlocking: true, maxHoldMinutes: 180 });
|
||
expect(s.maxFixRounds).toBe(5);
|
||
expect(s.followDays).toBe(1);
|
||
expect(s.autoRetry).toBe(false);
|
||
expect(s.enabled).toBe(true);
|
||
expect(s.freshness).toBe(true);
|
||
// 自动修复默认关:先用打回导入 / 一键修复跑顺
|
||
expect(s.autoFix).toBe(false);
|
||
expect("bugBlocking" in s).toBe(false);
|
||
});
|
||
});
|
||
|
||
describe("什么时候能修", () => {
|
||
const run = { outcome: null, phase: "awaiting" as const, fixRounds: 0, latestFindings: [
|
||
{ job: "check_t_zhongkui_pre", stage: "check", jobId: 1, url: "", category: "rule" as const, items: [] },
|
||
] };
|
||
test("红着等人、有 agent 能改的、次数没用完", () => {
|
||
expect(canFixNow(run, 2)).toBe(true);
|
||
expect(canFixNow({ ...run, fixRounds: 2 }, 2)).toBe(false);
|
||
expect(canFixNow({ ...run, phase: "fixing" }, 2)).toBe(false);
|
||
expect(canFixNow({ ...run, latestFindings: [{ ...run.latestFindings[0]!, category: "approval" }] }, 2)).toBe(false);
|
||
});
|
||
test("发现阶段:以「没有 MR」结束的那轮不算跟过 —— MR 后来补上了要重新接上", () => {
|
||
const entered = 1000;
|
||
expect(followedSince([], entered)).toBe(false);
|
||
expect(followedSince([{ outcome: null, endedAt: null }], entered)).toBe(true);
|
||
expect(followedSince([{ outcome: "green", endedAt: 2000 }], entered)).toBe(true);
|
||
expect(followedSince([{ outcome: "green", endedAt: 500 }], entered)).toBe(false);
|
||
expect(followedSince([{ outcome: "no_mr", endedAt: 2000 }], entered)).toBe(false);
|
||
});
|
||
test("跟的是复核中和复核通过后(MR 待合入)", () => {
|
||
expect(ciFollowStatuses("bug")).toEqual(["manual_review", "fix_succeeded"]);
|
||
expect(ciFollowStatuses("feature")).toEqual(["manual_review", "completed"]);
|
||
});
|
||
});
|
||
|
||
describe("收益 vs 成本", () => {
|
||
const base: CiGateRunView = {
|
||
id: "r", kind: "bug", itemId: "t", title: null, platform: "ios", mode: "follow", mrUrl: null, round: 1,
|
||
phase: "done", outcome: "green", startedAt: 1000, endedAt: null, deliveredAt: null, firstVerdict: "green",
|
||
firstVerdictAt: null, firstFailed: [], finalVerdict: "green", latestFindings: [], boostFrom: null,
|
||
fixRounds: 0, fixMs: 0, retries: 0, syncs: 0, events: [],
|
||
};
|
||
const MIN = 60_000;
|
||
test("首次全绿率 vs 结束时全绿率、红到绿用时、救回来的靠什么、回炉入口", () => {
|
||
const runs: CiGateRunView[] = [
|
||
{ ...base, id: "1" },
|
||
{ ...base, id: "2", firstVerdict: "red", firstVerdictAt: 2000, endedAt: 2000 + 90 * MIN, firstFailed: [{ job: "check_t_zhongkui_pre", category: "rule" }],
|
||
fixRounds: 1, fixMs: 50 * MIN, events: [{ at: 3000, type: "fix_requeue", source: "boost" }] },
|
||
{ ...base, id: "3", firstVerdict: "red", firstVerdictAt: 2000, endedAt: 2000 + 20 * MIN, firstFailed: [{ job: "build_m", category: "infra" }], retries: 1 },
|
||
// 人自己 push 修好、MR 合了:算绿,归「人修」
|
||
{ ...base, id: "4", firstVerdict: "red", firstVerdictAt: 2000, endedAt: 2000 + 300 * MIN, outcome: "merged", finalVerdict: "green", firstFailed: [{ job: "x", category: "approval" }] },
|
||
// 打回时导入:这一轮被新一轮取代,不进全绿率,但回炉入口计数
|
||
{ ...base, id: "5", firstVerdict: "red", outcome: "preempted", finalVerdict: "red", fixRounds: 1, events: [{ at: 3000, type: "fix_requeue", source: "reject" }] },
|
||
// 此刻红着等人
|
||
{ ...base, id: "6", outcome: null, phase: "awaiting", firstVerdict: "red", finalVerdict: null },
|
||
{ ...base, id: "7", kind: "feature" },
|
||
{ ...base, id: "8", startedAt: 0 },
|
||
];
|
||
const s = summarizeCiGate(runs, 500);
|
||
expect(s.bug.runs).toBe(6);
|
||
expect(s.bug.firstJudged).toBe(6);
|
||
expect(s.bug.firstGreen).toBe(1);
|
||
expect(s.bug.finalJudged).toBe(4);
|
||
expect(s.bug.finalGreen).toBe(4);
|
||
expect(s.bug.rescued).toBe(3);
|
||
expect(s.bug.rescuedBy).toEqual({ retry: 1, fix: 1, sync: 0, human: 1 });
|
||
expect(s.bug.redToGreenMedian).toBe(90 * MIN);
|
||
expect(s.bug.fixBySource).toEqual({ reject: 1, boost: 1, auto: 0 });
|
||
expect(s.bug.awaitingNow).toBe(1);
|
||
expect(s.bug.outcomes).toEqual({ green: 3, merged: 1, preempted: 1 });
|
||
expect(s.feature.runs).toBe(1);
|
||
});
|
||
});
|