Files
streamline/satellite/test/evidence_criteria_2609.test.ts
T
2026-11-16 17:26:44 -08:00

238 lines
9.9 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { describe, expect, test } from "bun:test";
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import {
BASELINE_CRITERIA,
evaluateCapture,
narrowedClaims,
pairMismatchIssue,
pairedEvidenceName,
splitProcessEvidence,
type CaptureCriteria,
type ManifestEntry,
type ProbeResult,
} from "@bbbug/shared/evidence_capture";
import {
listEvidenceImageNames,
listProcessImageNames,
readEvidencePremises,
} from "../src/evidence_names";
import { renderReadablePrompt } from "@bbbug/shared/readable_prompt";
/**
* 2026-09 证据审计 2.6「建议加进判定器 / 报告规则的判据」那张表的落地验收。
*
* 每个 describe 对应表里的一行。这些判据的共同点是:它们核的都不是文件属性
* ——文件属性早就核过了,26/30 的样本「有效」但其中 19 个带保留——而是
* **这份素材到底证明了声明的那件事没有**。所以测试也都盯在声明与素材的关系上,
* 而不是盯 ffprobe 的数字。
*/
/** v2-liveness:生产当前激活的那一版,新判据必须在它之上仍然成立。 */
const V2: CaptureCriteria = {
...BASELINE_CRITERIA,
version: 2, label: "v2-liveness",
deadEffectiveFps: 5, lowEffectiveFps: 15, aliveEffectiveFps: 15,
deadBytesPerSecond: 20480, failOnMissingDuration: true, staticRequiresLowLiveness: true,
};
/** 一段 54s / 171 帧的静态界面录制 —— 审计案例 6792056893 的形状。 */
const STILL_CLIP: ProbeResult = {
ok: true, sizeBytes: 3_000_000, durationS: 54, width: 1080, height: 2400,
nFrames: 171, codec: "h264", sceneChanges: 0,
};
describe("静态场景豁免(来源:5358966721、6792056893)", () => {
test("不声明时,合法的静止画面仍被判 fail —— 这正是审计抓到的误报", () => {
const { verdict, issues } = evaluateCapture("video", STILL_CLIP, V2);
expect(verdict).toBe("fail");
expect(issues.map((i) => i.code)).toContain("frozen");
});
test("声明了静止场景 → 降级为告警,证据保住、重录名额不被吃掉", () => {
const { verdict, issues } = evaluateCapture("video", STILL_CLIP, V2, "native", {
staticScene: "长按 @全部 的按压态,画面本就不动",
});
expect(verdict).toBe("warn");
// 判定没有消失,只是降级 —— 声明是 agent 的说法,不是测量结果。
const frozen = issues.find((i) => i.code === "frozen");
expect(frozen?.severity).toBe("warn");
expect(frozen?.message).toContain("已声明静止场景");
// 而且降级这件事本身留了一条带理由的记录,供复核的人反对。
const note = issues.find((i) => i.code === "static-scene-declared");
expect(note?.message).toContain("长按 @全部 的按压态");
});
test("声明不能把「根本没录成」洗白:空文件 / 损坏仍然 fail", () => {
const declared = { staticScene: "静态弹窗" };
expect(evaluateCapture("video", { ok: false, sizeBytes: 12 }, V2, "native", declared).verdict)
.toBe("fail");
expect(evaluateCapture("video", { ok: false, sizeBytes: 5_000_000 }, V2, "native", declared).verdict)
.toBe("fail");
});
test("声明了但本来就没问题 → 不加噪音,仍是 pass", () => {
const good: ProbeResult = {
ok: true, sizeBytes: 5_000_000, durationS: 20, width: 1080, height: 2400,
nFrames: 500, codec: "h264", sceneChanges: 9,
};
const { verdict, issues } = evaluateCapture("video", good, V2, "native", {
staticScene: "其实挺动的",
});
expect(verdict).toBe("pass");
expect(issues).toHaveLength(0);
});
});
describe("同场景成对校验(来源:6858399569、5313354254)", () => {
const after = (over: Partial<ManifestEntry> = {}): ManifestEntry => ({
v: 1, ts: 1, event: "shot", captureId: "c", out: "before-1-panel.png",
probe: { ok: true, sizeBytes: 9999, width: 1080, height: 2400 },
scene: "zh/弹窗",
...over,
});
test("before ⇄ after 互为配对名", () => {
expect(pairedEvidenceName("after-r2-story-tray.png")).toBe("before-r2-story-tray.png");
expect(pairedEvidenceName("before-1-panel.png")).toBe("after-1-panel.png");
expect(pairedEvidenceName("demo-1-x.mp4")).toBeNull();
});
test("中文页对英文页 → scene-mismatch(两张都通过了文件属性校验)", () => {
const issue = pairMismatchIssue(
{ out: "after-1-panel.png", scene: "en/弹窗", probe: { ok: true, sizeBytes: 9999, width: 1080, height: 2400 } },
after(),
);
expect(issue?.code).toBe("scene-mismatch");
expect(issue?.message).toContain("场景标识不同");
});
test("尺寸不同也算不同场景 —— 这条不依赖任何声明,换了机型就自动抓到", () => {
const issue = pairMismatchIssue(
{ out: "after-1-panel.png", probe: { ok: true, sizeBytes: 9999, width: 606, height: 1418 } },
after(),
);
expect(issue?.code).toBe("scene-mismatch");
expect(issue?.message).toContain("画面尺寸不同");
});
test("同场景 / 未声明 → 不报 —— 沉默不是不一致的证据", () => {
expect(pairMismatchIssue(
{ out: "after-1-panel.png", scene: "zh/弹窗", probe: { ok: true, sizeBytes: 1, width: 1080, height: 2400 } },
after(),
)).toBeNull();
expect(pairMismatchIssue(
{ out: "after-1-panel.png", probe: { ok: true, sizeBytes: 1 } },
after({ scene: undefined }),
)).toBeNull();
});
});
describe("probe / flow 与证据分离(来源:4724028576、5395707629)", () => {
test("probe-* 丢弃、flow-* 降级为过程记录、其余留在正区", () => {
const { evidence, process } = splitProcessEvidence([
"demo-1-fix.mp4", "before-1-x.png", "after-1-x.png",
"flow-1-01-login.png", "flow-1-02-logout.png", "probe-1-scroll.mp4", "mech-1-hook.png",
]);
expect(evidence).toEqual(["demo-1-fix.mp4", "before-1-x.png", "after-1-x.png", "mech-1-hook.png"]);
expect(process).toEqual(["flow-1-01-login.png", "flow-1-02-logout.png"]);
});
test("枚举证据目录时两条列表分开,probe-* 两边都不在", () => {
const dir = mkdtempSync(path.join(tmpdir(), "evid-split-"));
for (const n of ["before-1-x.png", "flow-1-login.png", "probe-1-scan.png"])
writeFileSync(path.join(dir, n), "x");
expect(listEvidenceImageNames(dir)).toEqual(["before-1-x.png"]);
expect(listProcessImageNames(dir)).toEqual(["flow-1-login.png"]);
});
test("过程记录进 prompt 时带着「不许当 hero / 不许当前后对比」的约束", () => {
const out = renderReadablePrompt("模板:{{evidenceImages}}", {
kind: "bug", title: "t", rawMarkdown: "x",
evidenceImages: ["before-1-x.png"],
processShots: ["flow-1-login.png"],
});
expect(out).toContain("flow-1-login.png");
expect(out).toContain("过程记录");
expect(out).toContain("不得");
});
test("没有过程记录就不追加空块 —— 空块会让模型去凑一段", () => {
const out = renderReadablePrompt("模板:{{evidenceImages}}", {
kind: "bug", title: "t", rawMarkdown: "x", evidenceImages: ["before-1-x.png"],
});
expect(out).not.toContain("过程记录(不是证据)");
});
});
describe("声明收窄标注(来源:5285234655、5412538175、4731039505)", () => {
const entry = (out: string, intent: string): ManifestEntry =>
({ v: 1, ts: 1, event: "shot", captureId: out, out, intent });
test("挑出前提被人为安排过的声明", () => {
const claims = narrowedClaims([
entry("mech-1-read.png", "用 Frida 注入把已读状态改成 2 条之后红点消失"),
entry("demo-1-ok.mp4", "点击入口后红点消失"),
entry("mech-2-src.png", "iOS 接收端没有设备,改用 Android 发送端 ext 里的 a:src 作为代理量"),
]);
expect(claims.map((c) => c.out)).toEqual(["mech-1-read.png", "mech-2-src.png"]);
expect(claims[0].terms).toContain("注入");
expect(claims[1].terms).toContain("代理");
});
test("重录后改成不带收窄词的声明 → 不再标注(最终说法才算数)", () => {
const claims = narrowedClaims([
entry("demo-1-x.mp4", "用 Frida 注入构造出该状态"),
entry("demo-1-x.mp4", "自然复现:连续滚动十余秒后卡顿出现"),
]);
expect(claims).toHaveLength(0);
});
test("AB 开关对比不算收窄 —— 否则这条标注会在多数安卓任务上常亮", () => {
expect(narrowedClaims([entry("demo-1-x.mp4", "关闭开关是旧行为,打开 flag 后新行为生效")]))
.toHaveLength(0);
});
test("从 manifest 读出来,渲染成报告 prompt 的「本证据成立的前提」并顶到首屏", () => {
const dir = mkdtempSync(path.join(tmpdir(), "evid-premise-"));
mkdirSync(dir, { recursive: true });
writeFileSync(path.join(dir, "manifest.jsonl"), [
JSON.stringify(entry("mech-1-read.png", "用 Frida 注入把已读状态改成 2 条")),
JSON.stringify(entry("demo-1-ok.mp4", "点击入口后红点消失")),
].join("\n") + "\n");
const premises = readEvidencePremises(dir);
expect(premises).toContain("mech-1-read.png");
expect(premises).not.toContain("demo-1-ok.mp4");
const out = renderReadablePrompt("模板", {
kind: "bug", title: "t", rawMarkdown: "x", evidencePremises: premises,
});
expect(out).toContain("本证据成立的前提");
expect(out).toContain("首屏");
expect(out).toContain("Frida 注入");
});
test("没有命中就不追加 —— 这是多数任务的常态", () => {
const dir = mkdtempSync(path.join(tmpdir(), "evid-premise-none-"));
writeFileSync(path.join(dir, "manifest.jsonl"),
JSON.stringify(entry("demo-1-ok.mp4", "点击入口后红点消失")) + "\n");
expect(readEvidencePremises(dir)).toBeNull();
expect(renderReadablePrompt("模板", { kind: "bug", title: "t", rawMarkdown: "x" }))
.not.toContain("本证据成立的前提");
});
});