Initial import of streamline

This commit is contained in:
2026-11-16 17:26:44 -08:00
commit 8e4aec1855
1221 changed files with 274256 additions and 0 deletions
+402
View File
@@ -0,0 +1,402 @@
import { describe, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import path from "node:path";
import {
checkpointCount, parseBitsMind, parseBitsPlanUrl, selectCases, testCasesPromptBlock, type TestCaseSuite,
} from "@bbbug/shared/test_cases";
import {
CASE_RUN_REPORT_FILE, CASE_RUN_REPORT_SKELETON, buildCaseRunData, caseShotName, caseVideoName,
caseVideosFromManifest, parseCaseRunReport, parseCaseShotName, parseCaseVideoName, parseVerdict,
renderCaseRunSummaryHtml, casePassRate,
} from "@bbbug/shared/case_run_report";
import type { ManifestEntry } from "@bbbug/shared/evidence_capture";
import { parseCaseRunVerdict, renderCaseRunVerdictPrompt } from "@bbbug/shared/case_run_verdict";
import {
CASE_EVIDENCE_NAME_EXAMPLES, DEFAULT_VERIFY_CASE_FIRST_TEMPLATE, DEFAULT_VERIFY_CASE_SECOND_TEMPLATE,
UI_AUDIT_REPORT_FILE, parseVerifySpec, verifyReportFile,
} from "@bbbug/shared/verify_prompt";
import type { Verification } from "@bbbug/shared/types";
import {
buildVerificationFirstPrompt, buildVerificationSecondPrompt, verificationPromptStage,
} from "../src/verify_runner";
/**
* 用例验证(验证线的基础格式)的三段契约:
* 1. Bits 用例树 → 用例 / 检查点(test_cases.ts)
* 2. prompt 里的报告骨架 ↔ 解析器(case_run_report.ts)
* 3. prompt 里的证据命名 ↔ 平台按名字对证据、给录像打点
* 任何一段漂了都是静默的:用例少了、报告解析不出来、截图对不上检查点、录像跳不到那一秒。
*/
const FIXTURE = JSON.parse(readFileSync(path.join(import.meta.dir, "fixtures/bits/share-to-story-mind.json"), "utf8"));
const PLAN_URL = "https://bits.bytedance.net/devops/1499964674/quality/plan/scenarios/planDetail/8324455/manualTask/14734032?devops_space_type=general&devops_space_old_id=66666782";
const SOURCE = { url: PLAN_URL, spaceId: 1499964674, testPlanId: 14734032, fetchedAt: 1, fetchedBy: "qa@example.com" };
const suite = (): TestCaseSuite => parseBitsMind(FIXTURE, SOURCE);
describe("Bits 用例树 → 用例", () => {
test("真实计划:叶子是检查点,前置 + 步骤是一条用例", () => {
const s = suite();
expect(s.title).toContain("Share to story");
expect(checkpointCount(s)).toBe(31);
expect(s.cases.length).toBe(18);
expect(s.cases.map((c) => c.id)).toEqual(Array.from({ length: 18 }, (_, i) => `C${String(i + 1).padStart(2, "0")}`));
// 检查点编号按先序遍历,跨用例连续
expect(s.cases.flatMap((c) => c.checkpoints.map((p) => p.no))).toEqual(Array.from({ length: 31 }, (_, i) => i + 1));
for (const c of s.cases) {
expect(c.pre).toBeTruthy();
expect(c.step).toBeTruthy();
expect(c.path.length).toBeGreaterThan(0);
for (const p of c.checkpoints) expect(p.leafId).toBeTruthy();
}
const first = s.cases[0];
expect(first.path.join(" › ")).toContain("v0 (control)");
expect(first.step).toContain("Tap Inbox");
expect(first.checkpoints.map((p) => p.expect)).toContain("Share-to-story editor opens successfully");
});
test("没打 nodeType 的老式脑图:叶子照样当预期,前置 / 步骤为空", () => {
const legacy = { data: { TestCase: [{ TestCaseTitle: "老计划", TestCaseMind: {
data: { text: "老计划" },
children: [{ data: { text: "登录" }, children: [{ data: { text: "进入首页" } }, { data: { text: "头像可见" } }] }],
} }] } };
const s = parseBitsMind(legacy, SOURCE);
expect(s.cases.length).toBe(1);
expect(s.cases[0].path).toEqual(["登录"]);
expect(s.cases[0].pre).toBeNull();
expect(s.cases[0].checkpoints.map((p) => p.expect)).toEqual(["进入首页", "头像可见"]);
});
test("空计划报错,而不是交一份零条用例的规约", () => {
expect(() => parseBitsMind({ data: { TestCase: [] } }, SOURCE)).toThrow();
});
test("取消勾选不重新编号", () => {
const s = selectCases(suite(), ["C02", "C05"]);
expect(s.cases.map((c) => c.id)).toEqual(["C02", "C05"]);
expect(s.excludedCheckpoints).toBe(31 - checkpointCount(s));
expect(s.cases[1].checkpoints[0].no).toBeGreaterThan(s.cases[0].checkpoints[0].no);
});
test("用例原文进 prompt,每个检查点都在", () => {
const block = testCasesPromptBlock(suite());
expect(block).toContain("18 条用例 · 31 个检查点");
expect(block).toContain("### C18");
for (let n = 1; n <= 31; n++) expect(block).toContain(`- #${n} `);
});
});
describe("Bits 链接", () => {
test("认执行页链接;空间 id 取路径里的,不取 devops_space_old_id", () => {
expect(parseBitsPlanUrl(PLAN_URL).ref).toEqual({ spaceId: 1499964674, testPlanId: 14734032 });
});
test("认不出的给出能照着改的原因", () => {
expect(parseBitsPlanUrl("https://bits.bytedance.net/devops/1/quality/case/caseDetail/2").reason).toContain("单条用例");
expect(parseBitsPlanUrl("https://bits.bytedance.net/devops/1/quality/plan/scenarios/planDetail/8").reason).toContain("manualTask");
expect(parseBitsPlanUrl("not a url").ref).toBeNull();
expect(parseBitsPlanUrl("").ref).toBeNull();
});
});
// ── 报告契约 ─────────────────────────────────────────────────────────────────
describe("报告骨架 ↔ 解析器", () => {
test("骨架原样能解析:一行结论 + 一行阻塞", () => {
const r = parseCaseRunReport(CASE_RUN_REPORT_SKELETON)!;
expect(r).not.toBeNull();
expect(r.rows).toHaveLength(1);
expect(r.rows[0]).toMatchObject({ no: 1, caseId: "C01", verdict: "pass" });
expect(r.blocked).toEqual([expect.objectContaining({ caseId: "C07" })]);
});
test("「不通过」不会被当成「通过」;四个词和容错写法都认", () => {
expect(parseVerdict("不通过")).toBe("fail");
expect(parseVerdict("**通过**")).toBe("pass");
expect(parseVerdict("阻塞")).toBe("blocked");
expect(parseVerdict("需人工")).toBe("judge");
expect(parseVerdict("FAIL")).toBe("fail");
expect(parseVerdict("看起来还行")).toBeNull();
});
test("没有「逐条结论」一节就不产出数据", () => {
expect(parseCaseRunReport("# 标题\n\n随便写的")).toBeNull();
});
test("同一检查点写了两行,后写的算数;附加章节原样保留", () => {
const md = `# t\n\n## 1. 逐条结论\n\n### C01\n\n| # | 结论 | 实际 | 细节 |\n|---|---|---|---|\n| 1 | 不通过 | 没弹 | — |\n| 1 | 通过 | 弹了 | 第 2 轮重跑 |\n\n## 2. 阻塞与局限\n\n| 用例 | 卡在哪 | 已做到哪一步 / 还差什么 |\n|---|---|---|\n\n## 第 2 轮复核处理\n\n- 意见 1 → 重跑 #1\n`;
const r = parseCaseRunReport(md)!;
expect(r.rows).toEqual([{ no: 1, caseId: "C01", verdict: "pass", actual: "弹了", detail: "第 2 轮重跑" }]);
expect(r.extra).toEqual([{ heading: "第 2 轮复核处理", body: "- 意见 1 → 重跑 #1" }]);
});
test("骨架外多写了列时按表头找结论列;反引号里的 | 不切列", () => {
// 7bca8ceb 的原样写法:多了「检查点」「证据」、没有「实际」,按位置取会整表判成未报告
const md = "# t\n\n## 1. 逐条结论\n\n### C01\n\n| # | 检查点 | 结论 | 证据 | 细节 |\n| --- | --- | --- | --- | --- |\n"
+ "| 1 | 编译无悬空引用 | 通过 | `cp01.png` | 用 `grep -E \"a|b\"` 过滤为空 |\n"
+ "| 2 | 带 ID 才还原 | **阻塞(协议项未验证)** | `cp02.png` | 缺 fixture |\n";
const r = parseCaseRunReport(md)!;
expect(r.rows).toEqual([
{ no: 1, caseId: "C01", verdict: "pass", actual: "", detail: "用 `grep -E \"a|b\"` 过滤为空" },
{ no: 2, caseId: "C01", verdict: "blocked", actual: "", detail: "缺 fixture" },
]);
});
test("两种格式的报告文件名各归各", () => {
expect(verifyReportFile("case-run")).toBe(CASE_RUN_REPORT_FILE);
expect(verifyReportFile("ui-audit")).toBe(UI_AUDIT_REPORT_FILE);
});
});
// ── 证据命名与录像打点 ─────────────────────────────────────────────────────────
describe("证据命名", () => {
test("prompt 里的示例名能被平台解析回同一个检查点", () => {
expect(parseCaseShotName(CASE_EVIDENCE_NAME_EXAMPLES.shot)).toEqual({ caseId: "C05", round: 1, no: 27 });
expect(parseCaseVideoName(CASE_EVIDENCE_NAME_EXAMPLES.video)).toEqual({ caseId: "C05", round: 1, part: 1 });
expect(parseCaseVideoName(CASE_EVIDENCE_NAME_EXAMPLES.videoPart2)).toEqual({ caseId: "C05", round: 1, part: 2 });
// 模板里写的是带 {{batchNo}} 的字面量 —— 代入第 1 轮必须和平台的命名函数一致
for (const tpl of [DEFAULT_VERIFY_CASE_FIRST_TEMPLATE, DEFAULT_VERIFY_CASE_SECOND_TEMPLATE]) {
const r1 = tpl.replaceAll("{{batchNo}}", "1");
expect(r1).toContain(caseShotName("C05", 1, 27));
expect(r1).toContain(caseVideoName("C05", 1));
expect(r1).toContain(caseVideoName("C05", 1, 2));
}
});
test("导航失败帧不算检查点截图", () => {
expect(parseCaseShotName("case-C05-r1-cp27.navfail.png")).toBeNull();
});
});
function entry(e: Partial<ManifestEntry> & Pick<ManifestEntry, "event" | "out" | "ts">): ManifestEntry {
return { v: 1, captureId: e.captureId ?? `${e.out}-${e.ts}`, ...e } as ManifestEntry;
}
describe("录像打点", () => {
const start = entry({ event: "start", out: "case-C05-r1.mp4", ts: 1_000, captureId: "cap1" });
// 挂钟 20s,文件里只有 19.2s —— 那 0.8s 花在录制器启动上,从偏移里扣掉
const stop = entry({ event: "stop", out: "case-C05-r1.mp4", ts: 21_000, captureId: "cap1", durationS: 20, probe: { ok: true, durationS: 19.2 } as ManifestEntry["probe"], verdict: "pass" });
test("录制期间拍的检查点截图 → 录像里的秒数(用 takenAt,扣掉启动延迟)", () => {
const vids = caseVideosFromManifest([
start,
entry({ event: "shot", out: "case-C05-r1-cp26.png", ts: 9_000, takenAt: 6_000 }),
entry({ event: "shot", out: "case-C05-r1-cp27.png", ts: 15_500 }), // 老 manifest 没有 takenAt → 退回 ts
entry({ event: "shot", out: "case-C05-r1-cp28.png", ts: 30_000, takenAt: 29_000 }), // 录制结束后才拍 → 不打点
entry({ event: "shot", out: "case-C06-r1-cp30.png", ts: 8_000, takenAt: 8_000 }), // 别的用例
stop,
]);
expect(vids).toHaveLength(1);
expect(vids[0]).toMatchObject({ file: "case-C05-r1.mp4", durationS: 19.2, verdict: "pass" });
expect(vids[0].marks).toEqual([{ no: 26, t: 4.2 }, { no: 27, t: 13.7 }]);
});
test("同名重录:只认最后一次 stop", () => {
const retry = [
entry({ event: "start", out: "case-C05-r1.mp4", ts: 1_000, captureId: "a" }),
entry({ event: "stop", out: "case-C05-r1.mp4", ts: 3_000, captureId: "a", durationS: 2, verdict: "fail" }),
entry({ event: "start", out: "case-C05-r1.mp4", ts: 10_000, captureId: "b" }),
entry({ event: "shot", out: "case-C05-r1-cp26.png", ts: 12_000, takenAt: 12_000 }),
entry({ event: "stop", out: "case-C05-r1.mp4", ts: 20_000, captureId: "b", durationS: 10, verdict: "pass" }),
];
const [v] = caseVideosFromManifest(retry);
expect(v.verdict).toBe("pass");
expect(v.marks).toEqual([{ no: 26, t: 2 }]);
});
});
// ── 拼数据 ──────────────────────────────────────────────────────────────────
describe("用例清单 + 报告 + 证据 → 面板数据", () => {
const s = suite();
const c = s.cases.find((x) => x.checkpoints.length >= 2)!;
const [a, b] = c.checkpoints;
const other = s.cases.find((x) => x.id !== c.id)!;
const md = `# 标题
| 项目 | 内容 |
|---|---|
| 被测包 | TikTok 38.1.0 (1234) |
## 1. 逐条结论
### ${c.id}
| # | 结论 | 实际 | 细节 |
|---|---|---|---|
| ${a.no} | 通过 | 正常打开 | — |
| ${b.no} | 不通过 | 没有副标题 | 复现 3/3 |
### ${other.id}
| # | 结论 | 实际 | 细节 |
|---|---|---|---|
${other.checkpoints.map((p) => `| ${p.no} | 阻塞 | 缺第二台设备 | — |`).join("\n")}
## 2. 阻塞与局限
| 用例 | 卡在哪 | 已做到哪一步 / 还差什么 |
|---|---|---|
| ${other.id} | 缺第二台设备 | 需要 Viewer 账号 |
`;
const report = parseCaseRunReport(md)!;
test("清单里有、报告里没写的检查点记「未报告」,不计为通过", () => {
const d = buildCaseRunData(s, report, { files: [], manifest: [] });
expect(d.format).toBe("case-run");
expect(d.cases).toHaveLength(18);
expect(d.totals.pass).toBe(1);
expect(d.totals.fail).toBe(1);
expect(d.totals.blocked).toBe(other.checkpoints.length);
expect(d.totals.missing).toBe(31 - 2 - other.checkpoints.length);
expect(casePassRate(d.totals)).toBe(50); // 阻塞 / 未报告不进分母
expect(d.cases.find((x) => x.id === other.id)!.blocked?.where).toBe("缺第二台设备");
expect(d.health.find((h) => h.name === "逐条结论")!.verdict).toBe("fail");
expect(d.meta).toEqual([{ label: "被测包", value: "TikTok 38.1.0 (1234)" }]);
});
test("截图按名字对到检查点,取最新一轮;不通过没录像 → 体检警告", () => {
const files = [caseShotName(c.id, 1, a.no), caseShotName(c.id, 1, b.no), caseShotName(c.id, 2, b.no)];
const d = buildCaseRunData(s, report, { files, manifest: [] });
const cc = d.cases.find((x) => x.id === c.id)!;
expect(cc.checkpoints[0].shot).toBe(caseShotName(c.id, 1, a.no));
expect(cc.checkpoints[1].shot).toBe(caseShotName(c.id, 2, b.no));
expect(d.health.find((h) => h.name === "不通过有复现录像")!.verdict).toBe("warn");
});
test("录像只放最新一轮;盘上没有的录像不列", () => {
const v1 = caseVideoName(c.id, 1), v2 = caseVideoName(c.id, 2);
const manifest = [
entry({ event: "start", out: v1, ts: 0, captureId: "x" }),
entry({ event: "stop", out: v1, ts: 5_000, captureId: "x", durationS: 5 }),
entry({ event: "start", out: v2, ts: 10_000, captureId: "y" }),
entry({ event: "shot", out: caseShotName(c.id, 2, b.no), ts: 12_000, takenAt: 12_000 }),
entry({ event: "stop", out: v2, ts: 20_000, captureId: "y", durationS: 10 }),
];
const d = buildCaseRunData(s, report, { files: [v1, v2, caseShotName(c.id, 2, b.no)], manifest });
const cc = d.cases.find((x) => x.id === c.id)!;
expect(cc.videos.map((v) => v.file)).toEqual([v2]);
expect(cc.videos[0].marks).toEqual([{ no: b.no, t: 2 }]);
expect(d.health.find((h) => h.name === "不通过有复现录像")!.verdict).toBe("pass");
const gone = buildCaseRunData(s, report, { files: [], manifest });
expect(gone.cases.find((x) => x.id === c.id)!.videos).toEqual([]);
});
test("没有用例清单时按报告还原骨架,不至于什么都看不到", () => {
const d = buildCaseRunData(null, report, { files: [], manifest: [] });
expect(d.cases.map((x) => x.id)).toEqual([c.id, other.id]);
expect(d.totals.missing).toBe(0);
});
test("降级摘要只列要看的检查点", () => {
const html = renderCaseRunSummaryHtml(buildCaseRunData(s, report, { files: [], manifest: [] }));
expect(html).toContain("不通过 1");
expect(html).toContain("没有副标题");
expect(html).not.toContain("正常打开");
});
});
// ── 规约与模板 ───────────────────────────────────────────────────────────────
function verification(over: Partial<Verification> = {}): Verification {
return {
id: "abcdef12-0000-0000-0000-000000000000", title: "S2S 默认样式", description: "用例验证",
platform: "ios", baseRef: null, remark: null, attachments: null,
verifySpec: JSON.stringify({ format: "case-run", scope: "v1 用强制分组账号", cases: suite() }),
reviewFeedback: null, reviewHistory: null, workspacePath: "/tmp/ws", pinned: 0,
status: "awaiting_verify", priorityKey: 0, watchers: [], createdAt: 0, startedAt: null,
finishedAt: null, durationMs: null, totalDurationMs: null, logPath: null, failureReason: null,
attempts: 0, executorNodeId: null, baseline: null, ...over,
};
}
describe("用例验证的规约与模板", () => {
test("没写 format 的老规约仍是 UI 走查", () => {
expect(parseVerifySpec(JSON.stringify({ scope: "x" }))!.format).toBe("ui-audit");
const s = parseVerifySpec(JSON.stringify({ format: "case-run", scope: "", cases: { bogus: 1 } }))!;
expect(s.format).toBe("case-run");
expect(s.cases).toBeUndefined();
});
test("按格式和轮次挑模板", () => {
const v = verification();
expect(verificationPromptStage(v, false)).toBe("verify.case.first");
expect(verificationPromptStage(v, true)).toBe("verify.case.second");
const ui = verification({ verifySpec: JSON.stringify({ format: "ui-audit", scope: "x" }) });
expect(verificationPromptStage(ui, false)).toBe("verify.run.first");
});
test("首轮 prompt:用例全文、补充说明、报告骨架都在;没有走查那一套", () => {
const p = buildVerificationFirstPrompt(verification(), "/tmp/ws");
expect(p).toContain(CASE_RUN_REPORT_FILE);
expect(p).toContain("v1 用强制分组账号");
expect(p).toContain("### C18");
expect(p).toContain("- #31 ");
expect(p).toContain("## 1. 逐条结论");
expect(p).toContain("设备上现在装着的那个");
expect(p).not.toContain(UI_AUDIT_REPORT_FILE);
expect(p).not.toContain("覆盖矩阵");
expect(p).not.toMatch(/\{\{\w+\}\}/); // 变量全部代入
});
test("指定了测试分支:要求核对装的包是不是这个分支出的,但不替它出包", () => {
const p = buildVerificationFirstPrompt(verification({ baseRef: "feature/s2s-fullscreen" }), "/tmp/ws");
expect(p).toContain("feature/s2s-fullscreen");
expect(p).toContain("不会");
});
test("补采轮在上一轮报告上改,新证据用新轮次命名", () => {
const p = buildVerificationSecondPrompt(verification({ attempts: 1 }), "/tmp/ws");
expect(p).toContain("第 2 轮");
expect(p).toContain("case-C05-r2-cp27.png");
expect(p).toContain("## 第 2 轮复核处理");
expect(p).not.toMatch(/\{\{\w+\}\}/);
});
test("控制面下发的正文优先于内置默认", () => {
const p = buildVerificationFirstPrompt(verification(), "/tmp/ws", "自定义 {{title}}");
expect(p.startsWith("自定义 S2S 默认样式")).toBe(true);
});
});
describe("结论卡收口", () => {
const md = "# t\n\n## 1. 逐条结论\n\n### C01\n\n| # | 结论 | 实际 | 细节 |\n|---|---|---|---|\n"
+ "| 1 | 通过 | 弹窗在 | — |\n| 2 | 通过 | 返回不关 | — |\n\n### C02\n\n| # | 结论 | 实际 | 细节 |\n|---|---|---|---|\n"
+ "| 3 | 阻塞 | 没账号 | — |\n| 4 | 阻塞 | 没账号 | — |\n";
const data = buildCaseRunData(null, parseCaseRunReport(md)!, { files: ["case-C01-r1-cp01.png"], manifest: [] });
const images = ["case-C01-r1-cp01.png", "case-C02-r1-cp03.navfail.png"];
test("检查点号只认这单真有、结论对得上的;截图只认证据目录里的;漏掉的没走到项补一条", () => {
const out = JSON.stringify({
status: "proven", headline: "返回键不再关掉弹窗", sub: "s", env: "e",
proven: [{ text: "按返回弹窗还在", cps: ["C01 #2", "#3", "C09 #99", "C01#1"], shots: [{ file: "case-C01-r1-cp01.png", cap: "前" }, { file: "made-up.png", cap: "x" }] }],
gaps: [{ cause: "没 Kids 账号", cps: ["C02 #3", "C01 #1"], what: "w", unblock: "给账号", owner: "QA" }],
findings: [{ kind: "weird", text: "留了测试数据" }], trust: [],
});
const v = parseCaseRunVerdict(`好的,如下:\n\`\`\`json\n${out}\n\`\`\``, data, images)!;
// 阻塞项没走到 → 不许说「已证实」
expect(v.status).toBe("partial");
expect(v.proven[0].cps).toEqual(["C01 #2", "C01 #1"]);
expect(v.proven[0].shots).toEqual([{ file: "case-C01-r1-cp01.png", cap: "前" }]);
expect(v.gaps[0].cps).toEqual(["C02 #3"]);
expect(v.gaps[1]).toMatchObject({ cause: "其它没走到的检查点", cps: ["C02 #4"] });
expect(v.findings[0].kind).toBe("note");
});
test("有不通过就是「发现问题」,模型说什么都不算;认不出的输出不出卡", () => {
const failed = buildCaseRunData(null, parseCaseRunReport(md.replace("| 2 | 通过 |", "| 2 | 不通过 |"))!, { files: [], manifest: [] });
const v = parseCaseRunVerdict(JSON.stringify({ status: "proven", headline: "h", proven: [], gaps: [] }), failed, [])!;
expect(v.status).toBe("failed");
expect(v.problems).toEqual([{ text: "其它不符合预期的检查点,见下方明细", cps: ["C01 #2"], why: undefined, shots: undefined }]);
expect(parseCaseRunVerdict("抱歉,我写不出来", data, images)).toBeNull();
expect(parseCaseRunVerdict(JSON.stringify({ status: "proven" }), data, images)).toBeNull();
});
test("提示词里列出逐条结论和可选截图,占位符都填上了", () => {
const p = renderCaseRunVerdictPrompt({ data, report: md, evidenceDir: "/w/evidence", images });
expect(p).toContain("C02 #3 [阻塞]");
expect(p).toContain("- case-C02-r1-cp03.navfail.png");
expect(p).not.toMatch(/\{\{\w+\}\}/);
});
});