import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import path from "node:path"; import { checkpointCount, parseBitsMind, parseBitsPlanUrl, selectCases, testCasesPromptBlock, type TestCaseSuite, } from "@bbbug/shared/test_cases"; import { CASE_RUN_REPORT_FILE, CASE_RUN_REPORT_SKELETON, buildCaseRunData, caseShotName, caseVideoName, caseVideosFromManifest, parseCaseRunReport, parseCaseShotName, parseCaseVideoName, parseVerdict, renderCaseRunSummaryHtml, casePassRate, } from "@bbbug/shared/case_run_report"; import type { ManifestEntry } from "@bbbug/shared/evidence_capture"; import { parseCaseRunVerdict, renderCaseRunVerdictPrompt } from "@bbbug/shared/case_run_verdict"; import { CASE_EVIDENCE_NAME_EXAMPLES, DEFAULT_VERIFY_CASE_FIRST_TEMPLATE, DEFAULT_VERIFY_CASE_SECOND_TEMPLATE, UI_AUDIT_REPORT_FILE, parseVerifySpec, verifyReportFile, } from "@bbbug/shared/verify_prompt"; import type { Verification } from "@bbbug/shared/types"; import { buildVerificationFirstPrompt, buildVerificationSecondPrompt, verificationPromptStage, } from "../src/verify_runner"; /** * 用例验证(验证线的基础格式)的三段契约: * 1. Bits 用例树 → 用例 / 检查点(test_cases.ts) * 2. prompt 里的报告骨架 ↔ 解析器(case_run_report.ts) * 3. prompt 里的证据命名 ↔ 平台按名字对证据、给录像打点 * 任何一段漂了都是静默的:用例少了、报告解析不出来、截图对不上检查点、录像跳不到那一秒。 */ const FIXTURE = JSON.parse(readFileSync(path.join(import.meta.dir, "fixtures/bits/share-to-story-mind.json"), "utf8")); const PLAN_URL = "https://bits.bytedance.net/devops/1499964674/quality/plan/scenarios/planDetail/8324455/manualTask/14734032?devops_space_type=general&devops_space_old_id=66666782"; const SOURCE = { url: PLAN_URL, spaceId: 1499964674, testPlanId: 14734032, fetchedAt: 1, fetchedBy: "qa@example.com" }; const suite = (): TestCaseSuite => parseBitsMind(FIXTURE, SOURCE); describe("Bits 用例树 → 用例", () => { test("真实计划:叶子是检查点,前置 + 步骤是一条用例", () => { const s = suite(); expect(s.title).toContain("Share to story"); expect(checkpointCount(s)).toBe(31); expect(s.cases.length).toBe(18); expect(s.cases.map((c) => c.id)).toEqual(Array.from({ length: 18 }, (_, i) => `C${String(i + 1).padStart(2, "0")}`)); // 检查点编号按先序遍历,跨用例连续 expect(s.cases.flatMap((c) => c.checkpoints.map((p) => p.no))).toEqual(Array.from({ length: 31 }, (_, i) => i + 1)); for (const c of s.cases) { expect(c.pre).toBeTruthy(); expect(c.step).toBeTruthy(); expect(c.path.length).toBeGreaterThan(0); for (const p of c.checkpoints) expect(p.leafId).toBeTruthy(); } const first = s.cases[0]; expect(first.path.join(" › ")).toContain("v0 (control)"); expect(first.step).toContain("Tap Inbox"); expect(first.checkpoints.map((p) => p.expect)).toContain("Share-to-story editor opens successfully"); }); test("没打 nodeType 的老式脑图:叶子照样当预期,前置 / 步骤为空", () => { const legacy = { data: { TestCase: [{ TestCaseTitle: "老计划", TestCaseMind: { data: { text: "老计划" }, children: [{ data: { text: "登录" }, children: [{ data: { text: "进入首页" } }, { data: { text: "头像可见" } }] }], } }] } }; const s = parseBitsMind(legacy, SOURCE); expect(s.cases.length).toBe(1); expect(s.cases[0].path).toEqual(["登录"]); expect(s.cases[0].pre).toBeNull(); expect(s.cases[0].checkpoints.map((p) => p.expect)).toEqual(["进入首页", "头像可见"]); }); test("空计划报错,而不是交一份零条用例的规约", () => { expect(() => parseBitsMind({ data: { TestCase: [] } }, SOURCE)).toThrow(); }); test("取消勾选不重新编号", () => { const s = selectCases(suite(), ["C02", "C05"]); expect(s.cases.map((c) => c.id)).toEqual(["C02", "C05"]); expect(s.excludedCheckpoints).toBe(31 - checkpointCount(s)); expect(s.cases[1].checkpoints[0].no).toBeGreaterThan(s.cases[0].checkpoints[0].no); }); test("用例原文进 prompt,每个检查点都在", () => { const block = testCasesPromptBlock(suite()); expect(block).toContain("18 条用例 · 31 个检查点"); expect(block).toContain("### C18"); for (let n = 1; n <= 31; n++) expect(block).toContain(`- #${n} `); }); }); describe("Bits 链接", () => { test("认执行页链接;空间 id 取路径里的,不取 devops_space_old_id", () => { expect(parseBitsPlanUrl(PLAN_URL).ref).toEqual({ spaceId: 1499964674, testPlanId: 14734032 }); }); test("认不出的给出能照着改的原因", () => { expect(parseBitsPlanUrl("https://bits.bytedance.net/devops/1/quality/case/caseDetail/2").reason).toContain("单条用例"); expect(parseBitsPlanUrl("https://bits.bytedance.net/devops/1/quality/plan/scenarios/planDetail/8").reason).toContain("manualTask"); expect(parseBitsPlanUrl("not a url").ref).toBeNull(); expect(parseBitsPlanUrl("").ref).toBeNull(); }); }); // ── 报告契约 ───────────────────────────────────────────────────────────────── describe("报告骨架 ↔ 解析器", () => { test("骨架原样能解析:一行结论 + 一行阻塞", () => { const r = parseCaseRunReport(CASE_RUN_REPORT_SKELETON)!; expect(r).not.toBeNull(); expect(r.rows).toHaveLength(1); expect(r.rows[0]).toMatchObject({ no: 1, caseId: "C01", verdict: "pass" }); expect(r.blocked).toEqual([expect.objectContaining({ caseId: "C07" })]); }); test("「不通过」不会被当成「通过」;四个词和容错写法都认", () => { expect(parseVerdict("不通过")).toBe("fail"); expect(parseVerdict("**通过**")).toBe("pass"); expect(parseVerdict("阻塞")).toBe("blocked"); expect(parseVerdict("需人工")).toBe("judge"); expect(parseVerdict("FAIL")).toBe("fail"); expect(parseVerdict("看起来还行")).toBeNull(); }); test("没有「逐条结论」一节就不产出数据", () => { expect(parseCaseRunReport("# 标题\n\n随便写的")).toBeNull(); }); test("同一检查点写了两行,后写的算数;附加章节原样保留", () => { const md = `# t\n\n## 1. 逐条结论\n\n### C01\n\n| # | 结论 | 实际 | 细节 |\n|---|---|---|---|\n| 1 | 不通过 | 没弹 | — |\n| 1 | 通过 | 弹了 | 第 2 轮重跑 |\n\n## 2. 阻塞与局限\n\n| 用例 | 卡在哪 | 已做到哪一步 / 还差什么 |\n|---|---|---|\n\n## 第 2 轮复核处理\n\n- 意见 1 → 重跑 #1\n`; const r = parseCaseRunReport(md)!; expect(r.rows).toEqual([{ no: 1, caseId: "C01", verdict: "pass", actual: "弹了", detail: "第 2 轮重跑" }]); expect(r.extra).toEqual([{ heading: "第 2 轮复核处理", body: "- 意见 1 → 重跑 #1" }]); }); test("骨架外多写了列时按表头找结论列;反引号里的 | 不切列", () => { // 7bca8ceb 的原样写法:多了「检查点」「证据」、没有「实际」,按位置取会整表判成未报告 const md = "# t\n\n## 1. 逐条结论\n\n### C01\n\n| # | 检查点 | 结论 | 证据 | 细节 |\n| --- | --- | --- | --- | --- |\n" + "| 1 | 编译无悬空引用 | 通过 | `cp01.png` | 用 `grep -E \"a|b\"` 过滤为空 |\n" + "| 2 | 带 ID 才还原 | **阻塞(协议项未验证)** | `cp02.png` | 缺 fixture |\n"; const r = parseCaseRunReport(md)!; expect(r.rows).toEqual([ { no: 1, caseId: "C01", verdict: "pass", actual: "", detail: "用 `grep -E \"a|b\"` 过滤为空" }, { no: 2, caseId: "C01", verdict: "blocked", actual: "", detail: "缺 fixture" }, ]); }); test("两种格式的报告文件名各归各", () => { expect(verifyReportFile("case-run")).toBe(CASE_RUN_REPORT_FILE); expect(verifyReportFile("ui-audit")).toBe(UI_AUDIT_REPORT_FILE); }); }); // ── 证据命名与录像打点 ───────────────────────────────────────────────────────── describe("证据命名", () => { test("prompt 里的示例名能被平台解析回同一个检查点", () => { expect(parseCaseShotName(CASE_EVIDENCE_NAME_EXAMPLES.shot)).toEqual({ caseId: "C05", round: 1, no: 27 }); expect(parseCaseVideoName(CASE_EVIDENCE_NAME_EXAMPLES.video)).toEqual({ caseId: "C05", round: 1, part: 1 }); expect(parseCaseVideoName(CASE_EVIDENCE_NAME_EXAMPLES.videoPart2)).toEqual({ caseId: "C05", round: 1, part: 2 }); // 模板里写的是带 {{batchNo}} 的字面量 —— 代入第 1 轮必须和平台的命名函数一致 for (const tpl of [DEFAULT_VERIFY_CASE_FIRST_TEMPLATE, DEFAULT_VERIFY_CASE_SECOND_TEMPLATE]) { const r1 = tpl.replaceAll("{{batchNo}}", "1"); expect(r1).toContain(caseShotName("C05", 1, 27)); expect(r1).toContain(caseVideoName("C05", 1)); expect(r1).toContain(caseVideoName("C05", 1, 2)); } }); test("导航失败帧不算检查点截图", () => { expect(parseCaseShotName("case-C05-r1-cp27.navfail.png")).toBeNull(); }); }); function entry(e: Partial & Pick): ManifestEntry { return { v: 1, captureId: e.captureId ?? `${e.out}-${e.ts}`, ...e } as ManifestEntry; } describe("录像打点", () => { const start = entry({ event: "start", out: "case-C05-r1.mp4", ts: 1_000, captureId: "cap1" }); // 挂钟 20s,文件里只有 19.2s —— 那 0.8s 花在录制器启动上,从偏移里扣掉 const stop = entry({ event: "stop", out: "case-C05-r1.mp4", ts: 21_000, captureId: "cap1", durationS: 20, probe: { ok: true, durationS: 19.2 } as ManifestEntry["probe"], verdict: "pass" }); test("录制期间拍的检查点截图 → 录像里的秒数(用 takenAt,扣掉启动延迟)", () => { const vids = caseVideosFromManifest([ start, entry({ event: "shot", out: "case-C05-r1-cp26.png", ts: 9_000, takenAt: 6_000 }), entry({ event: "shot", out: "case-C05-r1-cp27.png", ts: 15_500 }), // 老 manifest 没有 takenAt → 退回 ts entry({ event: "shot", out: "case-C05-r1-cp28.png", ts: 30_000, takenAt: 29_000 }), // 录制结束后才拍 → 不打点 entry({ event: "shot", out: "case-C06-r1-cp30.png", ts: 8_000, takenAt: 8_000 }), // 别的用例 stop, ]); expect(vids).toHaveLength(1); expect(vids[0]).toMatchObject({ file: "case-C05-r1.mp4", durationS: 19.2, verdict: "pass" }); expect(vids[0].marks).toEqual([{ no: 26, t: 4.2 }, { no: 27, t: 13.7 }]); }); test("同名重录:只认最后一次 stop", () => { const retry = [ entry({ event: "start", out: "case-C05-r1.mp4", ts: 1_000, captureId: "a" }), entry({ event: "stop", out: "case-C05-r1.mp4", ts: 3_000, captureId: "a", durationS: 2, verdict: "fail" }), entry({ event: "start", out: "case-C05-r1.mp4", ts: 10_000, captureId: "b" }), entry({ event: "shot", out: "case-C05-r1-cp26.png", ts: 12_000, takenAt: 12_000 }), entry({ event: "stop", out: "case-C05-r1.mp4", ts: 20_000, captureId: "b", durationS: 10, verdict: "pass" }), ]; const [v] = caseVideosFromManifest(retry); expect(v.verdict).toBe("pass"); expect(v.marks).toEqual([{ no: 26, t: 2 }]); }); }); // ── 拼数据 ────────────────────────────────────────────────────────────────── describe("用例清单 + 报告 + 证据 → 面板数据", () => { const s = suite(); const c = s.cases.find((x) => x.checkpoints.length >= 2)!; const [a, b] = c.checkpoints; const other = s.cases.find((x) => x.id !== c.id)!; const md = `# 标题 | 项目 | 内容 | |---|---| | 被测包 | TikTok 38.1.0 (1234) | ## 1. 逐条结论 ### ${c.id} | # | 结论 | 实际 | 细节 | |---|---|---|---| | ${a.no} | 通过 | 正常打开 | — | | ${b.no} | 不通过 | 没有副标题 | 复现 3/3 | ### ${other.id} | # | 结论 | 实际 | 细节 | |---|---|---|---| ${other.checkpoints.map((p) => `| ${p.no} | 阻塞 | 缺第二台设备 | — |`).join("\n")} ## 2. 阻塞与局限 | 用例 | 卡在哪 | 已做到哪一步 / 还差什么 | |---|---|---| | ${other.id} | 缺第二台设备 | 需要 Viewer 账号 | `; const report = parseCaseRunReport(md)!; test("清单里有、报告里没写的检查点记「未报告」,不计为通过", () => { const d = buildCaseRunData(s, report, { files: [], manifest: [] }); expect(d.format).toBe("case-run"); expect(d.cases).toHaveLength(18); expect(d.totals.pass).toBe(1); expect(d.totals.fail).toBe(1); expect(d.totals.blocked).toBe(other.checkpoints.length); expect(d.totals.missing).toBe(31 - 2 - other.checkpoints.length); expect(casePassRate(d.totals)).toBe(50); // 阻塞 / 未报告不进分母 expect(d.cases.find((x) => x.id === other.id)!.blocked?.where).toBe("缺第二台设备"); expect(d.health.find((h) => h.name === "逐条结论")!.verdict).toBe("fail"); expect(d.meta).toEqual([{ label: "被测包", value: "TikTok 38.1.0 (1234)" }]); }); test("截图按名字对到检查点,取最新一轮;不通过没录像 → 体检警告", () => { const files = [caseShotName(c.id, 1, a.no), caseShotName(c.id, 1, b.no), caseShotName(c.id, 2, b.no)]; const d = buildCaseRunData(s, report, { files, manifest: [] }); const cc = d.cases.find((x) => x.id === c.id)!; expect(cc.checkpoints[0].shot).toBe(caseShotName(c.id, 1, a.no)); expect(cc.checkpoints[1].shot).toBe(caseShotName(c.id, 2, b.no)); expect(d.health.find((h) => h.name === "不通过有复现录像")!.verdict).toBe("warn"); }); test("录像只放最新一轮;盘上没有的录像不列", () => { const v1 = caseVideoName(c.id, 1), v2 = caseVideoName(c.id, 2); const manifest = [ entry({ event: "start", out: v1, ts: 0, captureId: "x" }), entry({ event: "stop", out: v1, ts: 5_000, captureId: "x", durationS: 5 }), entry({ event: "start", out: v2, ts: 10_000, captureId: "y" }), entry({ event: "shot", out: caseShotName(c.id, 2, b.no), ts: 12_000, takenAt: 12_000 }), entry({ event: "stop", out: v2, ts: 20_000, captureId: "y", durationS: 10 }), ]; const d = buildCaseRunData(s, report, { files: [v1, v2, caseShotName(c.id, 2, b.no)], manifest }); const cc = d.cases.find((x) => x.id === c.id)!; expect(cc.videos.map((v) => v.file)).toEqual([v2]); expect(cc.videos[0].marks).toEqual([{ no: b.no, t: 2 }]); expect(d.health.find((h) => h.name === "不通过有复现录像")!.verdict).toBe("pass"); const gone = buildCaseRunData(s, report, { files: [], manifest }); expect(gone.cases.find((x) => x.id === c.id)!.videos).toEqual([]); }); test("没有用例清单时按报告还原骨架,不至于什么都看不到", () => { const d = buildCaseRunData(null, report, { files: [], manifest: [] }); expect(d.cases.map((x) => x.id)).toEqual([c.id, other.id]); expect(d.totals.missing).toBe(0); }); test("降级摘要只列要看的检查点", () => { const html = renderCaseRunSummaryHtml(buildCaseRunData(s, report, { files: [], manifest: [] })); expect(html).toContain("不通过 1"); expect(html).toContain("没有副标题"); expect(html).not.toContain("正常打开"); }); }); // ── 规约与模板 ─────────────────────────────────────────────────────────────── function verification(over: Partial = {}): Verification { return { id: "abcdef12-0000-0000-0000-000000000000", title: "S2S 默认样式", description: "用例验证", platform: "ios", baseRef: null, remark: null, attachments: null, verifySpec: JSON.stringify({ format: "case-run", scope: "v1 用强制分组账号", cases: suite() }), reviewFeedback: null, reviewHistory: null, workspacePath: "/tmp/ws", pinned: 0, status: "awaiting_verify", priorityKey: 0, watchers: [], createdAt: 0, startedAt: null, finishedAt: null, durationMs: null, totalDurationMs: null, logPath: null, failureReason: null, attempts: 0, executorNodeId: null, baseline: null, ...over, }; } describe("用例验证的规约与模板", () => { test("没写 format 的老规约仍是 UI 走查", () => { expect(parseVerifySpec(JSON.stringify({ scope: "x" }))!.format).toBe("ui-audit"); const s = parseVerifySpec(JSON.stringify({ format: "case-run", scope: "", cases: { bogus: 1 } }))!; expect(s.format).toBe("case-run"); expect(s.cases).toBeUndefined(); }); test("按格式和轮次挑模板", () => { const v = verification(); expect(verificationPromptStage(v, false)).toBe("verify.case.first"); expect(verificationPromptStage(v, true)).toBe("verify.case.second"); const ui = verification({ verifySpec: JSON.stringify({ format: "ui-audit", scope: "x" }) }); expect(verificationPromptStage(ui, false)).toBe("verify.run.first"); }); test("首轮 prompt:用例全文、补充说明、报告骨架都在;没有走查那一套", () => { const p = buildVerificationFirstPrompt(verification(), "/tmp/ws"); expect(p).toContain(CASE_RUN_REPORT_FILE); expect(p).toContain("v1 用强制分组账号"); expect(p).toContain("### C18"); expect(p).toContain("- #31 "); expect(p).toContain("## 1. 逐条结论"); expect(p).toContain("设备上现在装着的那个"); expect(p).not.toContain(UI_AUDIT_REPORT_FILE); expect(p).not.toContain("覆盖矩阵"); expect(p).not.toMatch(/\{\{\w+\}\}/); // 变量全部代入 }); test("指定了测试分支:要求核对装的包是不是这个分支出的,但不替它出包", () => { const p = buildVerificationFirstPrompt(verification({ baseRef: "feature/s2s-fullscreen" }), "/tmp/ws"); expect(p).toContain("feature/s2s-fullscreen"); expect(p).toContain("不会"); }); test("补采轮在上一轮报告上改,新证据用新轮次命名", () => { const p = buildVerificationSecondPrompt(verification({ attempts: 1 }), "/tmp/ws"); expect(p).toContain("第 2 轮"); expect(p).toContain("case-C05-r2-cp27.png"); expect(p).toContain("## 第 2 轮复核处理"); expect(p).not.toMatch(/\{\{\w+\}\}/); }); test("控制面下发的正文优先于内置默认", () => { const p = buildVerificationFirstPrompt(verification(), "/tmp/ws", "自定义 {{title}}"); expect(p.startsWith("自定义 S2S 默认样式")).toBe(true); }); }); describe("结论卡收口", () => { const md = "# t\n\n## 1. 逐条结论\n\n### C01\n\n| # | 结论 | 实际 | 细节 |\n|---|---|---|---|\n" + "| 1 | 通过 | 弹窗在 | — |\n| 2 | 通过 | 返回不关 | — |\n\n### C02\n\n| # | 结论 | 实际 | 细节 |\n|---|---|---|---|\n" + "| 3 | 阻塞 | 没账号 | — |\n| 4 | 阻塞 | 没账号 | — |\n"; const data = buildCaseRunData(null, parseCaseRunReport(md)!, { files: ["case-C01-r1-cp01.png"], manifest: [] }); const images = ["case-C01-r1-cp01.png", "case-C02-r1-cp03.navfail.png"]; test("检查点号只认这单真有、结论对得上的;截图只认证据目录里的;漏掉的没走到项补一条", () => { const out = JSON.stringify({ status: "proven", headline: "返回键不再关掉弹窗", sub: "s", env: "e", proven: [{ text: "按返回弹窗还在", cps: ["C01 #2", "#3", "C09 #99", "C01#1"], shots: [{ file: "case-C01-r1-cp01.png", cap: "前" }, { file: "made-up.png", cap: "x" }] }], gaps: [{ cause: "没 Kids 账号", cps: ["C02 #3", "C01 #1"], what: "w", unblock: "给账号", owner: "QA" }], findings: [{ kind: "weird", text: "留了测试数据" }], trust: [], }); const v = parseCaseRunVerdict(`好的,如下:\n\`\`\`json\n${out}\n\`\`\``, data, images)!; // 阻塞项没走到 → 不许说「已证实」 expect(v.status).toBe("partial"); expect(v.proven[0].cps).toEqual(["C01 #2", "C01 #1"]); expect(v.proven[0].shots).toEqual([{ file: "case-C01-r1-cp01.png", cap: "前" }]); expect(v.gaps[0].cps).toEqual(["C02 #3"]); expect(v.gaps[1]).toMatchObject({ cause: "其它没走到的检查点", cps: ["C02 #4"] }); expect(v.findings[0].kind).toBe("note"); }); test("有不通过就是「发现问题」,模型说什么都不算;认不出的输出不出卡", () => { const failed = buildCaseRunData(null, parseCaseRunReport(md.replace("| 2 | 通过 |", "| 2 | 不通过 |"))!, { files: [], manifest: [] }); const v = parseCaseRunVerdict(JSON.stringify({ status: "proven", headline: "h", proven: [], gaps: [] }), failed, [])!; expect(v.status).toBe("failed"); expect(v.problems).toEqual([{ text: "其它不符合预期的检查点,见下方明细", cps: ["C01 #2"], why: undefined, shots: undefined }]); expect(parseCaseRunVerdict("抱歉,我写不出来", data, images)).toBeNull(); expect(parseCaseRunVerdict(JSON.stringify({ status: "proven" }), data, images)).toBeNull(); }); test("提示词里列出逐条结论和可选截图,占位符都填上了", () => { const p = renderCaseRunVerdictPrompt({ data, report: md, evidenceDir: "/w/evidence", images }); expect(p).toContain("C02 #3 [阻塞]"); expect(p).toContain("- case-C02-r1-cp03.navfail.png"); expect(p).not.toMatch(/\{\{\w+\}\}/); }); });