Files
streamline/satellite/test/case_run.test.ts
T
2026-11-16 17:26:44 -08:00

403 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { describe, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import path from "node:path";
import {
checkpointCount, parseBitsMind, parseBitsPlanUrl, selectCases, testCasesPromptBlock, type TestCaseSuite,
} from "@bbbug/shared/test_cases";
import {
CASE_RUN_REPORT_FILE, CASE_RUN_REPORT_SKELETON, buildCaseRunData, caseShotName, caseVideoName,
caseVideosFromManifest, parseCaseRunReport, parseCaseShotName, parseCaseVideoName, parseVerdict,
renderCaseRunSummaryHtml, casePassRate,
} from "@bbbug/shared/case_run_report";
import type { ManifestEntry } from "@bbbug/shared/evidence_capture";
import { parseCaseRunVerdict, renderCaseRunVerdictPrompt } from "@bbbug/shared/case_run_verdict";
import {
CASE_EVIDENCE_NAME_EXAMPLES, DEFAULT_VERIFY_CASE_FIRST_TEMPLATE, DEFAULT_VERIFY_CASE_SECOND_TEMPLATE,
UI_AUDIT_REPORT_FILE, parseVerifySpec, verifyReportFile,
} from "@bbbug/shared/verify_prompt";
import type { Verification } from "@bbbug/shared/types";
import {
buildVerificationFirstPrompt, buildVerificationSecondPrompt, verificationPromptStage,
} from "../src/verify_runner";
/**
* 用例验证(验证线的基础格式)的三段契约:
* 1. Bits 用例树 → 用例 / 检查点(test_cases.ts)
* 2. prompt 里的报告骨架 ↔ 解析器(case_run_report.ts)
* 3. prompt 里的证据命名 ↔ 平台按名字对证据、给录像打点
* 任何一段漂了都是静默的:用例少了、报告解析不出来、截图对不上检查点、录像跳不到那一秒。
*/
const FIXTURE = JSON.parse(readFileSync(path.join(import.meta.dir, "fixtures/bits/share-to-story-mind.json"), "utf8"));
const PLAN_URL = "https://bits.bytedance.net/devops/1499964674/quality/plan/scenarios/planDetail/8324455/manualTask/14734032?devops_space_type=general&devops_space_old_id=66666782";
const SOURCE = { url: PLAN_URL, spaceId: 1499964674, testPlanId: 14734032, fetchedAt: 1, fetchedBy: "qa@example.com" };
const suite = (): TestCaseSuite => parseBitsMind(FIXTURE, SOURCE);
describe("Bits 用例树 → 用例", () => {
test("真实计划:叶子是检查点,前置 + 步骤是一条用例", () => {
const s = suite();
expect(s.title).toContain("Share to story");
expect(checkpointCount(s)).toBe(31);
expect(s.cases.length).toBe(18);
expect(s.cases.map((c) => c.id)).toEqual(Array.from({ length: 18 }, (_, i) => `C${String(i + 1).padStart(2, "0")}`));
// 检查点编号按先序遍历,跨用例连续
expect(s.cases.flatMap((c) => c.checkpoints.map((p) => p.no))).toEqual(Array.from({ length: 31 }, (_, i) => i + 1));
for (const c of s.cases) {
expect(c.pre).toBeTruthy();
expect(c.step).toBeTruthy();
expect(c.path.length).toBeGreaterThan(0);
for (const p of c.checkpoints) expect(p.leafId).toBeTruthy();
}
const first = s.cases[0];
expect(first.path.join(" › ")).toContain("v0 (control)");
expect(first.step).toContain("Tap Inbox");
expect(first.checkpoints.map((p) => p.expect)).toContain("Share-to-story editor opens successfully");
});
test("没打 nodeType 的老式脑图:叶子照样当预期,前置 / 步骤为空", () => {
const legacy = { data: { TestCase: [{ TestCaseTitle: "老计划", TestCaseMind: {
data: { text: "老计划" },
children: [{ data: { text: "登录" }, children: [{ data: { text: "进入首页" } }, { data: { text: "头像可见" } }] }],
} }] } };
const s = parseBitsMind(legacy, SOURCE);
expect(s.cases.length).toBe(1);
expect(s.cases[0].path).toEqual(["登录"]);
expect(s.cases[0].pre).toBeNull();
expect(s.cases[0].checkpoints.map((p) => p.expect)).toEqual(["进入首页", "头像可见"]);
});
test("空计划报错,而不是交一份零条用例的规约", () => {
expect(() => parseBitsMind({ data: { TestCase: [] } }, SOURCE)).toThrow();
});
test("取消勾选不重新编号", () => {
const s = selectCases(suite(), ["C02", "C05"]);
expect(s.cases.map((c) => c.id)).toEqual(["C02", "C05"]);
expect(s.excludedCheckpoints).toBe(31 - checkpointCount(s));
expect(s.cases[1].checkpoints[0].no).toBeGreaterThan(s.cases[0].checkpoints[0].no);
});
test("用例原文进 prompt,每个检查点都在", () => {
const block = testCasesPromptBlock(suite());
expect(block).toContain("18 条用例 · 31 个检查点");
expect(block).toContain("### C18");
for (let n = 1; n <= 31; n++) expect(block).toContain(`- #${n} `);
});
});
describe("Bits 链接", () => {
test("认执行页链接;空间 id 取路径里的,不取 devops_space_old_id", () => {
expect(parseBitsPlanUrl(PLAN_URL).ref).toEqual({ spaceId: 1499964674, testPlanId: 14734032 });
});
test("认不出的给出能照着改的原因", () => {
expect(parseBitsPlanUrl("https://bits.bytedance.net/devops/1/quality/case/caseDetail/2").reason).toContain("单条用例");
expect(parseBitsPlanUrl("https://bits.bytedance.net/devops/1/quality/plan/scenarios/planDetail/8").reason).toContain("manualTask");
expect(parseBitsPlanUrl("not a url").ref).toBeNull();
expect(parseBitsPlanUrl("").ref).toBeNull();
});
});
// ── 报告契约 ─────────────────────────────────────────────────────────────────
describe("报告骨架 ↔ 解析器", () => {
test("骨架原样能解析:一行结论 + 一行阻塞", () => {
const r = parseCaseRunReport(CASE_RUN_REPORT_SKELETON)!;
expect(r).not.toBeNull();
expect(r.rows).toHaveLength(1);
expect(r.rows[0]).toMatchObject({ no: 1, caseId: "C01", verdict: "pass" });
expect(r.blocked).toEqual([expect.objectContaining({ caseId: "C07" })]);
});
test("「不通过」不会被当成「通过」;四个词和容错写法都认", () => {
expect(parseVerdict("不通过")).toBe("fail");
expect(parseVerdict("**通过**")).toBe("pass");
expect(parseVerdict("阻塞")).toBe("blocked");
expect(parseVerdict("需人工")).toBe("judge");
expect(parseVerdict("FAIL")).toBe("fail");
expect(parseVerdict("看起来还行")).toBeNull();
});
test("没有「逐条结论」一节就不产出数据", () => {
expect(parseCaseRunReport("# 标题\n\n随便写的")).toBeNull();
});
test("同一检查点写了两行,后写的算数;附加章节原样保留", () => {
const md = `# t\n\n## 1. 逐条结论\n\n### C01\n\n| # | 结论 | 实际 | 细节 |\n|---|---|---|---|\n| 1 | 不通过 | 没弹 | — |\n| 1 | 通过 | 弹了 | 第 2 轮重跑 |\n\n## 2. 阻塞与局限\n\n| 用例 | 卡在哪 | 已做到哪一步 / 还差什么 |\n|---|---|---|\n\n## 第 2 轮复核处理\n\n- 意见 1 → 重跑 #1\n`;
const r = parseCaseRunReport(md)!;
expect(r.rows).toEqual([{ no: 1, caseId: "C01", verdict: "pass", actual: "弹了", detail: "第 2 轮重跑" }]);
expect(r.extra).toEqual([{ heading: "第 2 轮复核处理", body: "- 意见 1 → 重跑 #1" }]);
});
test("骨架外多写了列时按表头找结论列;反引号里的 | 不切列", () => {
// 7bca8ceb 的原样写法:多了「检查点」「证据」、没有「实际」,按位置取会整表判成未报告
const md = "# t\n\n## 1. 逐条结论\n\n### C01\n\n| # | 检查点 | 结论 | 证据 | 细节 |\n| --- | --- | --- | --- | --- |\n"
+ "| 1 | 编译无悬空引用 | 通过 | `cp01.png` | 用 `grep -E \"a|b\"` 过滤为空 |\n"
+ "| 2 | 带 ID 才还原 | **阻塞(协议项未验证)** | `cp02.png` | 缺 fixture |\n";
const r = parseCaseRunReport(md)!;
expect(r.rows).toEqual([
{ no: 1, caseId: "C01", verdict: "pass", actual: "", detail: "用 `grep -E \"a|b\"` 过滤为空" },
{ no: 2, caseId: "C01", verdict: "blocked", actual: "", detail: "缺 fixture" },
]);
});
test("两种格式的报告文件名各归各", () => {
expect(verifyReportFile("case-run")).toBe(CASE_RUN_REPORT_FILE);
expect(verifyReportFile("ui-audit")).toBe(UI_AUDIT_REPORT_FILE);
});
});
// ── 证据命名与录像打点 ─────────────────────────────────────────────────────────
describe("证据命名", () => {
test("prompt 里的示例名能被平台解析回同一个检查点", () => {
expect(parseCaseShotName(CASE_EVIDENCE_NAME_EXAMPLES.shot)).toEqual({ caseId: "C05", round: 1, no: 27 });
expect(parseCaseVideoName(CASE_EVIDENCE_NAME_EXAMPLES.video)).toEqual({ caseId: "C05", round: 1, part: 1 });
expect(parseCaseVideoName(CASE_EVIDENCE_NAME_EXAMPLES.videoPart2)).toEqual({ caseId: "C05", round: 1, part: 2 });
// 模板里写的是带 {{batchNo}} 的字面量 —— 代入第 1 轮必须和平台的命名函数一致
for (const tpl of [DEFAULT_VERIFY_CASE_FIRST_TEMPLATE, DEFAULT_VERIFY_CASE_SECOND_TEMPLATE]) {
const r1 = tpl.replaceAll("{{batchNo}}", "1");
expect(r1).toContain(caseShotName("C05", 1, 27));
expect(r1).toContain(caseVideoName("C05", 1));
expect(r1).toContain(caseVideoName("C05", 1, 2));
}
});
test("导航失败帧不算检查点截图", () => {
expect(parseCaseShotName("case-C05-r1-cp27.navfail.png")).toBeNull();
});
});
function entry(e: Partial<ManifestEntry> & Pick<ManifestEntry, "event" | "out" | "ts">): ManifestEntry {
return { v: 1, captureId: e.captureId ?? `${e.out}-${e.ts}`, ...e } as ManifestEntry;
}
describe("录像打点", () => {
const start = entry({ event: "start", out: "case-C05-r1.mp4", ts: 1_000, captureId: "cap1" });
// 挂钟 20s,文件里只有 19.2s —— 那 0.8s 花在录制器启动上,从偏移里扣掉
const stop = entry({ event: "stop", out: "case-C05-r1.mp4", ts: 21_000, captureId: "cap1", durationS: 20, probe: { ok: true, durationS: 19.2 } as ManifestEntry["probe"], verdict: "pass" });
test("录制期间拍的检查点截图 → 录像里的秒数(用 takenAt,扣掉启动延迟)", () => {
const vids = caseVideosFromManifest([
start,
entry({ event: "shot", out: "case-C05-r1-cp26.png", ts: 9_000, takenAt: 6_000 }),
entry({ event: "shot", out: "case-C05-r1-cp27.png", ts: 15_500 }), // 老 manifest 没有 takenAt → 退回 ts
entry({ event: "shot", out: "case-C05-r1-cp28.png", ts: 30_000, takenAt: 29_000 }), // 录制结束后才拍 → 不打点
entry({ event: "shot", out: "case-C06-r1-cp30.png", ts: 8_000, takenAt: 8_000 }), // 别的用例
stop,
]);
expect(vids).toHaveLength(1);
expect(vids[0]).toMatchObject({ file: "case-C05-r1.mp4", durationS: 19.2, verdict: "pass" });
expect(vids[0].marks).toEqual([{ no: 26, t: 4.2 }, { no: 27, t: 13.7 }]);
});
test("同名重录:只认最后一次 stop", () => {
const retry = [
entry({ event: "start", out: "case-C05-r1.mp4", ts: 1_000, captureId: "a" }),
entry({ event: "stop", out: "case-C05-r1.mp4", ts: 3_000, captureId: "a", durationS: 2, verdict: "fail" }),
entry({ event: "start", out: "case-C05-r1.mp4", ts: 10_000, captureId: "b" }),
entry({ event: "shot", out: "case-C05-r1-cp26.png", ts: 12_000, takenAt: 12_000 }),
entry({ event: "stop", out: "case-C05-r1.mp4", ts: 20_000, captureId: "b", durationS: 10, verdict: "pass" }),
];
const [v] = caseVideosFromManifest(retry);
expect(v.verdict).toBe("pass");
expect(v.marks).toEqual([{ no: 26, t: 2 }]);
});
});
// ── 拼数据 ──────────────────────────────────────────────────────────────────
describe("用例清单 + 报告 + 证据 → 面板数据", () => {
const s = suite();
const c = s.cases.find((x) => x.checkpoints.length >= 2)!;
const [a, b] = c.checkpoints;
const other = s.cases.find((x) => x.id !== c.id)!;
const md = `# 标题
| 项目 | 内容 |
|---|---|
| 被测包 | TikTok 38.1.0 (1234) |
## 1. 逐条结论
### ${c.id}
| # | 结论 | 实际 | 细节 |
|---|---|---|---|
| ${a.no} | 通过 | 正常打开 | — |
| ${b.no} | 不通过 | 没有副标题 | 复现 3/3 |
### ${other.id}
| # | 结论 | 实际 | 细节 |
|---|---|---|---|
${other.checkpoints.map((p) => `| ${p.no} | 阻塞 | 缺第二台设备 | — |`).join("\n")}
## 2. 阻塞与局限
| 用例 | 卡在哪 | 已做到哪一步 / 还差什么 |
|---|---|---|
| ${other.id} | 缺第二台设备 | 需要 Viewer 账号 |
`;
const report = parseCaseRunReport(md)!;
test("清单里有、报告里没写的检查点记「未报告」,不计为通过", () => {
const d = buildCaseRunData(s, report, { files: [], manifest: [] });
expect(d.format).toBe("case-run");
expect(d.cases).toHaveLength(18);
expect(d.totals.pass).toBe(1);
expect(d.totals.fail).toBe(1);
expect(d.totals.blocked).toBe(other.checkpoints.length);
expect(d.totals.missing).toBe(31 - 2 - other.checkpoints.length);
expect(casePassRate(d.totals)).toBe(50); // 阻塞 / 未报告不进分母
expect(d.cases.find((x) => x.id === other.id)!.blocked?.where).toBe("缺第二台设备");
expect(d.health.find((h) => h.name === "逐条结论")!.verdict).toBe("fail");
expect(d.meta).toEqual([{ label: "被测包", value: "TikTok 38.1.0 (1234)" }]);
});
test("截图按名字对到检查点,取最新一轮;不通过没录像 → 体检警告", () => {
const files = [caseShotName(c.id, 1, a.no), caseShotName(c.id, 1, b.no), caseShotName(c.id, 2, b.no)];
const d = buildCaseRunData(s, report, { files, manifest: [] });
const cc = d.cases.find((x) => x.id === c.id)!;
expect(cc.checkpoints[0].shot).toBe(caseShotName(c.id, 1, a.no));
expect(cc.checkpoints[1].shot).toBe(caseShotName(c.id, 2, b.no));
expect(d.health.find((h) => h.name === "不通过有复现录像")!.verdict).toBe("warn");
});
test("录像只放最新一轮;盘上没有的录像不列", () => {
const v1 = caseVideoName(c.id, 1), v2 = caseVideoName(c.id, 2);
const manifest = [
entry({ event: "start", out: v1, ts: 0, captureId: "x" }),
entry({ event: "stop", out: v1, ts: 5_000, captureId: "x", durationS: 5 }),
entry({ event: "start", out: v2, ts: 10_000, captureId: "y" }),
entry({ event: "shot", out: caseShotName(c.id, 2, b.no), ts: 12_000, takenAt: 12_000 }),
entry({ event: "stop", out: v2, ts: 20_000, captureId: "y", durationS: 10 }),
];
const d = buildCaseRunData(s, report, { files: [v1, v2, caseShotName(c.id, 2, b.no)], manifest });
const cc = d.cases.find((x) => x.id === c.id)!;
expect(cc.videos.map((v) => v.file)).toEqual([v2]);
expect(cc.videos[0].marks).toEqual([{ no: b.no, t: 2 }]);
expect(d.health.find((h) => h.name === "不通过有复现录像")!.verdict).toBe("pass");
const gone = buildCaseRunData(s, report, { files: [], manifest });
expect(gone.cases.find((x) => x.id === c.id)!.videos).toEqual([]);
});
test("没有用例清单时按报告还原骨架,不至于什么都看不到", () => {
const d = buildCaseRunData(null, report, { files: [], manifest: [] });
expect(d.cases.map((x) => x.id)).toEqual([c.id, other.id]);
expect(d.totals.missing).toBe(0);
});
test("降级摘要只列要看的检查点", () => {
const html = renderCaseRunSummaryHtml(buildCaseRunData(s, report, { files: [], manifest: [] }));
expect(html).toContain("不通过 1");
expect(html).toContain("没有副标题");
expect(html).not.toContain("正常打开");
});
});
// ── 规约与模板 ───────────────────────────────────────────────────────────────
function verification(over: Partial<Verification> = {}): Verification {
return {
id: "abcdef12-0000-0000-0000-000000000000", title: "S2S 默认样式", description: "用例验证",
platform: "ios", baseRef: null, remark: null, attachments: null,
verifySpec: JSON.stringify({ format: "case-run", scope: "v1 用强制分组账号", cases: suite() }),
reviewFeedback: null, reviewHistory: null, workspacePath: "/tmp/ws", pinned: 0,
status: "awaiting_verify", priorityKey: 0, watchers: [], createdAt: 0, startedAt: null,
finishedAt: null, durationMs: null, totalDurationMs: null, logPath: null, failureReason: null,
attempts: 0, executorNodeId: null, baseline: null, ...over,
};
}
describe("用例验证的规约与模板", () => {
test("没写 format 的老规约仍是 UI 走查", () => {
expect(parseVerifySpec(JSON.stringify({ scope: "x" }))!.format).toBe("ui-audit");
const s = parseVerifySpec(JSON.stringify({ format: "case-run", scope: "", cases: { bogus: 1 } }))!;
expect(s.format).toBe("case-run");
expect(s.cases).toBeUndefined();
});
test("按格式和轮次挑模板", () => {
const v = verification();
expect(verificationPromptStage(v, false)).toBe("verify.case.first");
expect(verificationPromptStage(v, true)).toBe("verify.case.second");
const ui = verification({ verifySpec: JSON.stringify({ format: "ui-audit", scope: "x" }) });
expect(verificationPromptStage(ui, false)).toBe("verify.run.first");
});
test("首轮 prompt:用例全文、补充说明、报告骨架都在;没有走查那一套", () => {
const p = buildVerificationFirstPrompt(verification(), "/tmp/ws");
expect(p).toContain(CASE_RUN_REPORT_FILE);
expect(p).toContain("v1 用强制分组账号");
expect(p).toContain("### C18");
expect(p).toContain("- #31 ");
expect(p).toContain("## 1. 逐条结论");
expect(p).toContain("设备上现在装着的那个");
expect(p).not.toContain(UI_AUDIT_REPORT_FILE);
expect(p).not.toContain("覆盖矩阵");
expect(p).not.toMatch(/\{\{\w+\}\}/); // 变量全部代入
});
test("指定了测试分支:要求核对装的包是不是这个分支出的,但不替它出包", () => {
const p = buildVerificationFirstPrompt(verification({ baseRef: "feature/s2s-fullscreen" }), "/tmp/ws");
expect(p).toContain("feature/s2s-fullscreen");
expect(p).toContain("不会");
});
test("补采轮在上一轮报告上改,新证据用新轮次命名", () => {
const p = buildVerificationSecondPrompt(verification({ attempts: 1 }), "/tmp/ws");
expect(p).toContain("第 2 轮");
expect(p).toContain("case-C05-r2-cp27.png");
expect(p).toContain("## 第 2 轮复核处理");
expect(p).not.toMatch(/\{\{\w+\}\}/);
});
test("控制面下发的正文优先于内置默认", () => {
const p = buildVerificationFirstPrompt(verification(), "/tmp/ws", "自定义 {{title}}");
expect(p.startsWith("自定义 S2S 默认样式")).toBe(true);
});
});
describe("结论卡收口", () => {
const md = "# t\n\n## 1. 逐条结论\n\n### C01\n\n| # | 结论 | 实际 | 细节 |\n|---|---|---|---|\n"
+ "| 1 | 通过 | 弹窗在 | — |\n| 2 | 通过 | 返回不关 | — |\n\n### C02\n\n| # | 结论 | 实际 | 细节 |\n|---|---|---|---|\n"
+ "| 3 | 阻塞 | 没账号 | — |\n| 4 | 阻塞 | 没账号 | — |\n";
const data = buildCaseRunData(null, parseCaseRunReport(md)!, { files: ["case-C01-r1-cp01.png"], manifest: [] });
const images = ["case-C01-r1-cp01.png", "case-C02-r1-cp03.navfail.png"];
test("检查点号只认这单真有、结论对得上的;截图只认证据目录里的;漏掉的没走到项补一条", () => {
const out = JSON.stringify({
status: "proven", headline: "返回键不再关掉弹窗", sub: "s", env: "e",
proven: [{ text: "按返回弹窗还在", cps: ["C01 #2", "#3", "C09 #99", "C01#1"], shots: [{ file: "case-C01-r1-cp01.png", cap: "前" }, { file: "made-up.png", cap: "x" }] }],
gaps: [{ cause: "没 Kids 账号", cps: ["C02 #3", "C01 #1"], what: "w", unblock: "给账号", owner: "QA" }],
findings: [{ kind: "weird", text: "留了测试数据" }], trust: [],
});
const v = parseCaseRunVerdict(`好的,如下:\n\`\`\`json\n${out}\n\`\`\``, data, images)!;
// 阻塞项没走到 → 不许说「已证实」
expect(v.status).toBe("partial");
expect(v.proven[0].cps).toEqual(["C01 #2", "C01 #1"]);
expect(v.proven[0].shots).toEqual([{ file: "case-C01-r1-cp01.png", cap: "前" }]);
expect(v.gaps[0].cps).toEqual(["C02 #3"]);
expect(v.gaps[1]).toMatchObject({ cause: "其它没走到的检查点", cps: ["C02 #4"] });
expect(v.findings[0].kind).toBe("note");
});
test("有不通过就是「发现问题」,模型说什么都不算;认不出的输出不出卡", () => {
const failed = buildCaseRunData(null, parseCaseRunReport(md.replace("| 2 | 通过 |", "| 2 | 不通过 |"))!, { files: [], manifest: [] });
const v = parseCaseRunVerdict(JSON.stringify({ status: "proven", headline: "h", proven: [], gaps: [] }), failed, [])!;
expect(v.status).toBe("failed");
expect(v.problems).toEqual([{ text: "其它不符合预期的检查点,见下方明细", cps: ["C01 #2"], why: undefined, shots: undefined }]);
expect(parseCaseRunVerdict("抱歉,我写不出来", data, images)).toBeNull();
expect(parseCaseRunVerdict(JSON.stringify({ status: "proven" }), data, images)).toBeNull();
});
test("提示词里列出逐条结论和可选截图,占位符都填上了", () => {
const p = renderCaseRunVerdictPrompt({ data, report: md, evidenceDir: "/w/evidence", images });
expect(p).toContain("C02 #3 [阻塞]");
expect(p).toContain("- case-C02-r1-cp03.navfail.png");
expect(p).not.toMatch(/\{\{\w+\}\}/);
});
});