331 lines
17 KiB
TypeScript
331 lines
17 KiB
TypeScript
import {
|
|
LOOP, computeRoundMetrics, commandFamily, isWaitCommand, isBlindSleep, roundFailed,
|
|
unitHash, lintCandidate, placeholders,
|
|
offlineGate, judgeLab, isLabTaskId, LAB_ID_PREFIX, type TimingLine, type LabPair,
|
|
computeEvidenceMetrics, aggregateRounds, roundFixed, LOOP_METRICS, lintMetricGaming, type EvidenceEntry,
|
|
labLoadGateOf, pausedWithin,
|
|
} from "@bbbug/shared/rsi_loop";
|
|
|
|
const DAY = 86_400_000;
|
|
|
|
describe("过程指标", () => {
|
|
test("等界面状态 / 盲等 / 失败 / 连续同类失败 / 墙钟", () => {
|
|
const lines: TimingLine[] = [
|
|
{ t: "Bash", ms: 1000, gap: 500, ok: true, a: "adb shell input tap 1 2; sleep 3" },
|
|
{ t: "Bash", ms: 300_000, gap: 2000, ok: true, a: "until adb shell dumpsys window | grep -q mCurrentFocus; do sleep 5; done" },
|
|
{ t: "Bash", ms: 60_000, gap: 1000, ok: true, a: "sleep 60; adb shell dumpsys activity activities" },
|
|
{ t: "Bash", ms: 120_000, gap: 1000, ok: false, a: "bun satellite.js wait-for --build x" },
|
|
{ t: "Bash", ms: 2000, gap: 1000, ok: false, a: "cd /tmp && frida -U -p 123 -l a.js" },
|
|
{ t: "Bash", ms: 2000, gap: 1000, ok: false, a: "frida -U -n app -l b.js" },
|
|
{ t: "Read", ms: 10, gap: 100, ok: true, a: "{}" },
|
|
];
|
|
const m = computeRoundMetrics(lines, { status: "STATUS: FIXED", completed: true });
|
|
expect(m.ui_wait_s).toBe(360); // until 循环 + sleep 60 那条;点击后 sleep 3 不算
|
|
expect(m.blind_sleep_s).toBe(360); // wait-for 不算盲等
|
|
expect(m.failed_calls).toBe(3);
|
|
expect(m.repeat_fail_calls).toBe(1); // 第二次 frida 失败
|
|
expect(m.frida_fail_calls).toBe(2);
|
|
expect(m.tool_calls).toBe(7);
|
|
expect(m.round_wall_s).toBeCloseTo((1000 + 300_000 + 60_000 + 120_000 + 2000 + 2000 + 10 + 500 + 2000 + 1000 + 1000 + 1000 + 1000 + 100) / 1000, 5);
|
|
expect(m.round_failed).toBe(0);
|
|
});
|
|
|
|
test("等待命令的边界", () => {
|
|
expect(isWaitCommand("adb shell input tap 1 2; sleep 4; adb shell uiautomator dump")).toBe(false);
|
|
expect(isWaitCommand("sleep 15")).toBe(true);
|
|
expect(isWaitCommand("for i in 1 2 3; do x; sleep 2; done")).toBe(true);
|
|
expect(isBlindSleep("bun x.js wait-for --until 'true'")).toBe(false);
|
|
});
|
|
|
|
test("失败批次:不可复现不算失败,未收尾算失败", () => {
|
|
expect(roundFailed({ status: "STATUS: UNREPRODUCIBLE", completed: true })).toBe(false);
|
|
expect(roundFailed({ status: "STATUS: FIXED", completed: true })).toBe(false);
|
|
expect(roundFailed({ status: "STATUS: IMPLEMENTATION_FAILED", completed: true })).toBe(true);
|
|
expect(roundFailed({ status: null, completed: true })).toBe(true);
|
|
expect(roundFailed({ status: "STATUS: FIXED", completed: false })).toBe(true);
|
|
});
|
|
|
|
test("命令族归一化", () => {
|
|
expect(commandFamily("Bash", "cd /x/y && nohup frida -U -p 1 -l /a/b.js &")).toBe("frida");
|
|
expect(commandFamily("Bash", "A=1 adb -s SER shell input tap 1 2")).toBe("adb shell input");
|
|
expect(commandFamily("mcp__actl__actl_click", "{}")).toBe("mcp__actl__actl_click");
|
|
});
|
|
});
|
|
|
|
describe("确定性抽样", () => {
|
|
test("同一键同一值,不同键不同值,分布大致均匀", () => {
|
|
expect(unitHash("e1|t1")).toBe(unitHash("e1|t1"));
|
|
expect(unitHash("a")).not.toBe(unitHash("b"));
|
|
let k = 0;
|
|
for (let i = 0; i < 5000; i++) if (unitHash(`exp-x|${i}`) < 0.2) k++;
|
|
expect(k / 5000).toBeGreaterThan(0.17);
|
|
expect(k / 5000).toBeLessThan(0.23);
|
|
});
|
|
});
|
|
|
|
describe("候选模板校验", () => {
|
|
const base = "第一行 {{workspace}}\n第二行\n第三行 {{round}}";
|
|
test("只插入 → 通过", () => {
|
|
expect(lintCandidate(base, "第一行 {{workspace}}\n新增纪律\n第二行\n第三行 {{round}}\n结尾补充")).toBeNull();
|
|
});
|
|
test("丢了占位符 / 改了已有行 / 删了内容 → 拒绝", () => {
|
|
expect(lintCandidate(base, "第一行\n第二行\n第三行 {{round}}")).toMatch(/占位符/);
|
|
expect(lintCandidate(base, "第一行 {{workspace}}\n第二行改了一些字\n第三行 {{round}}")).toMatch(/已有行/);
|
|
expect(lintCandidate(base, "第一行 {{workspace}}\n第三行 {{round}}")).not.toBeNull();
|
|
});
|
|
test("占位符提取", () => {
|
|
expect(placeholders("{{ a }} {{b.c}} {{a}}")).toEqual(["a", "b.c"]);
|
|
});
|
|
});
|
|
|
|
describe("离线闸(不能明显变差)", () => {
|
|
const flat = { points: 24, seeds: 2, baseBad: 0.19, candBad: 0.21, diffMean: -0.02, ciLow: -0.08, ciHigh: 0.10 };
|
|
test("没显著变好但也没明显变差 → 放行去真机复跑", () => {
|
|
expect(offlineGate({ target: flat, holdout: flat })).toBeNull();
|
|
});
|
|
test("目标类或保留集下界低于 −0.10 → 拦下", () => {
|
|
expect(offlineGate({ target: { ...flat, ciLow: -0.13 }, holdout: flat })).toMatch(/目标类可能明显变差/);
|
|
expect(offlineGate({ target: flat, holdout: { ...flat, ciLow: -0.27 } })).toMatch(/保留集/);
|
|
});
|
|
test("点数不足 / 没有结果 → 拦下", () => {
|
|
expect(offlineGate({ target: { ...flat, points: 10 }, holdout: flat })).toMatch(/不足/);
|
|
expect(offlineGate(null)).not.toBeNull();
|
|
});
|
|
});
|
|
|
|
describe("真机复跑", () => {
|
|
const T0 = Date.UTC(2026, 9, 2);
|
|
const run = (status: string | null, ui: number, wall: number) => ({ status, metrics: { ui_wait_s: ui, round_wall_s: wall } });
|
|
const pair = (i: number, base: ReturnType<typeof run>, cand: ReturnType<typeof run>): LabPair =>
|
|
({ sourceId: `s${i}`, base, candidate: cand });
|
|
|
|
test("复跑任务按 id 前缀识别", () => {
|
|
expect(isLabTaskId(`${LAB_ID_PREFIX}abc`)).toBe(true);
|
|
expect(isLabTaskId("evaluate-123")).toBe(false);
|
|
expect(isLabTaskId(null)).toBe(false);
|
|
});
|
|
|
|
test("还有没跑完的 → 继续", () => {
|
|
const pairs = [pair(0, run("manual_review", 300, 3600), run(null, 0, 0))];
|
|
expect(judgeLab("ui_wait_s", pairs, T0, T0 + 3_600_000).decision).toBe("continue");
|
|
});
|
|
|
|
test("目标改善、修好数不少、用时没变长 → 通过", () => {
|
|
const pairs = [0, 1, 2, 3, 4].map((i) => pair(i, run("manual_review", 300, 3600), run("manual_review", 120, 3500)));
|
|
const j = judgeLab("ui_wait_s", pairs, T0, T0 + 86_400_000);
|
|
expect(j.decision).toBe("pass");
|
|
expect(j.candFixed).toBe(5);
|
|
});
|
|
|
|
test("目标没改善 / 修好变少 / 用时长了超过 20% → 不通过,并写明原因", () => {
|
|
const worse = [0, 1, 2].map((i) => pair(i, run("manual_review", 100, 3600), run("manual_review", 200, 3600)));
|
|
expect(judgeLab("ui_wait_s", worse, T0, T0 + 1).reason).toMatch(/没有改善/);
|
|
const fewer = [0, 1, 2].map((i) => pair(i, run("manual_review", 300, 3600), run(i === 0 ? "fix_failed" : "manual_review", 100, 3600)));
|
|
expect(judgeLab("ui_wait_s", fewer, T0, T0 + 1).reason).toMatch(/修好的个数变少/);
|
|
const slow = [0, 1, 2].map((i) => pair(i, run("manual_review", 300, 3600), run("manual_review", 100, 5000)));
|
|
expect(judgeLab("ui_wait_s", slow, T0, T0 + 1).reason).toMatch(/用时变长/);
|
|
});
|
|
|
|
test("到期只跑完不到 3 对 → 不通过(判不了)", () => {
|
|
const pairs = [
|
|
pair(0, run("manual_review", 300, 3600), run("manual_review", 100, 3600)),
|
|
pair(1, run(null, 0, 0), run(null, 0, 0)),
|
|
pair(2, run(null, 0, 0), run(null, 0, 0)),
|
|
];
|
|
const j = judgeLab("ui_wait_s", pairs, T0, T0 + (LOOP.LAB_MAX_DAYS + 1) * DAY);
|
|
expect(j.decision).toBe("fail");
|
|
expect(j.reason).toMatch(/判不了/);
|
|
});
|
|
});
|
|
|
|
describe("业务指标", () => {
|
|
const S = Date.UTC(2026, 9, 2, 8), MIN = 60_000;
|
|
const shot = (ts: number, phase: string, verdict = "pass", event = "shot"): EvidenceEntry => ({ ts, event, phase, verdict });
|
|
|
|
test("到达 = 第一张校验通过的 before 证据;fail 的不算,start 不算", () => {
|
|
const m = computeEvidenceMetrics([
|
|
{ ts: S + 5 * MIN, event: "start", phase: "before" },
|
|
shot(S + 10 * MIN, "before", "fail"),
|
|
shot(S + 20 * MIN, "before", "warn", "stop"),
|
|
shot(S + 30 * MIN, "before"),
|
|
shot(S + 50 * MIN, "after"),
|
|
], S, S + 60 * MIN);
|
|
expect(m.reach_s).toBe(20 * 60);
|
|
expect(m.reach_miss).toBe(0);
|
|
expect(m.evidence_fail).toBe(1);
|
|
});
|
|
|
|
test("没拍到合格 before → 未到达,用时按整批截尾", () => {
|
|
const m = computeEvidenceMetrics([shot(S + 10 * MIN, "before", "fail"), shot(S + 20 * MIN, "flow")], S, S + 40 * MIN);
|
|
expect(m.reach_miss).toBe(1);
|
|
expect(m.reach_s).toBe(40 * 60);
|
|
expect(computeEvidenceMetrics([], S, S + 40 * MIN).reach_miss).toBe(1);
|
|
});
|
|
|
|
test("按时间窗归批,不按条目的 round 字段;之前批次已到达的,这批不计到达", () => {
|
|
const entries = [shot(S + 10 * MIN, "before"), shot(S + 200 * MIN, "after", "fail")];
|
|
const r2 = computeEvidenceMetrics(entries, S + 120 * MIN, S + 240 * MIN);
|
|
expect(r2.reach_s).toBeUndefined();
|
|
expect(r2.reach_miss).toBeUndefined();
|
|
expect(r2.evidence_fail).toBe(1);
|
|
});
|
|
|
|
test("走不通时长:只有 FIXED 记 0", () => {
|
|
const lines: TimingLine[] = [{ t: "Bash", ms: 60_000, gap: 0, ok: true, a: "ls" }];
|
|
expect(computeRoundMetrics(lines, { status: "STATUS: FIXED", completed: true }).dead_end_s).toBe(0);
|
|
expect(computeRoundMetrics(lines, { status: "STATUS: UNREPRODUCIBLE", completed: true }).dead_end_s).toBe(60);
|
|
expect(computeRoundMetrics(lines, { status: "STATUS: FIX_FAILED", completed: true }).dead_end_s).toBe(60);
|
|
expect(computeRoundMetrics(lines, { status: null, completed: false }).dead_end_s).toBe(60);
|
|
expect(roundFixed({ status: "STATUS: UNFIXABLE", completed: true })).toBe(false);
|
|
});
|
|
|
|
test("任务级汇总:到达取各批最小、其余求和", () => {
|
|
const agg = aggregateRounds([
|
|
{ reach_s: 3600, reach_miss: 1, evidence_fail: 1, round_wall_s: 3600 },
|
|
{ reach_s: 600, reach_miss: 0, evidence_fail: 0, round_wall_s: 1800 },
|
|
{ evidence_fail: 2, round_wall_s: 900 },
|
|
]);
|
|
expect(agg.reach_miss).toBe(0);
|
|
expect(agg.reach_s).toBe(4200);
|
|
expect(agg.evidence_fail).toBe(3);
|
|
expect(agg.round_wall_s).toBe(6300);
|
|
});
|
|
|
|
test("业务指标都能被提案登记为目标", () => {
|
|
for (const k of ["reach_s", "reach_miss", "evidence_fail", "dead_end_s"]) {
|
|
const d = LOOP_METRICS.find((m) => m.key === k);
|
|
expect(d?.role).toBe("target");
|
|
expect(d?.group).toBe("business");
|
|
}
|
|
});
|
|
|
|
test("复跑底线:到达复现现场的个数不得少于现行(防跳过复现换提速)", () => {
|
|
const T0 = Date.UTC(2026, 9, 2);
|
|
const arm = (reach_miss: number, reach_s: number) => ({ status: "manual_review", metrics: { reach_s, reach_miss, round_wall_s: 3600 } });
|
|
const pairs: LabPair[] = [0, 1, 2].map((i) => ({ sourceId: `s${i}`, base: arm(0, 1800), candidate: arm(i === 0 ? 1 : 0, 600) }));
|
|
const j = judgeLab("reach_s", pairs, T0, T0 + 1);
|
|
expect(j.decision).toBe("fail");
|
|
expect(j.reason).toMatch(/到达复现现场的个数变少/);
|
|
expect(j.reached).toEqual({ base: 3, candidate: 2, n: 3 });
|
|
const ok: LabPair[] = [0, 1, 2].map((i) => ({ sourceId: `s${i}`, base: arm(0, 1800), candidate: arm(0, 900) }));
|
|
expect(judgeLab("reach_s", ok, T0, T0 + 1).decision).toBe("pass");
|
|
});
|
|
|
|
test("到达类目标的候选不许改证据采集;证据类目标不受限", () => {
|
|
const base = "# 步骤 3\n复现问题。\n# 步骤 4";
|
|
const shoot = "# 步骤 3\n复现问题。\n- 启动 App 后立刻用 record-evidence 拍一张 before-1-x.png。\n# 步骤 4";
|
|
const route = "# 步骤 3\n复现问题。\n- 先读 TestSpec 的入口路径,用 deeplink 直达目标页,不要从首页逐级点。\n# 步骤 4";
|
|
expect(lintMetricGaming("reach_s", base, shoot)).toMatch(/证据采集/);
|
|
expect(lintMetricGaming("reach_miss", base, shoot)).toMatch(/证据采集/);
|
|
expect(lintMetricGaming("reach_s", base, route)).toBeNull();
|
|
expect(lintMetricGaming("evidence_fail", base, shoot)).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe("执行配置包", () => {
|
|
test("只收目录里的旋钮和取值;坏的一律当空(= 生产行为)", async () => {
|
|
const { validateProfile, parseProfile, profileCliArgs, profilePromptBlock, profileLabel } = await import("@bbbug/shared/executor_profile");
|
|
expect(validateProfile({ effort: "medium", ui_settle: "on" })).toEqual({ profile: { effort: "medium", ui_settle: "on" } });
|
|
expect("error" in validateProfile({ effort: "turbo" })).toBe(true);
|
|
expect("error" in validateProfile({ model: "x" })).toBe(true);
|
|
expect("error" in validateProfile(["effort"])).toBe(true);
|
|
expect(parseProfile('{"effort":"turbo"}')).toEqual({});
|
|
expect(parseProfile("not json")).toEqual({});
|
|
expect(parseProfile(null)).toEqual({});
|
|
expect(profileCliArgs({ effort: "low" })).toEqual(["--effort=low"]);
|
|
expect(profileCliArgs({})).toEqual([]);
|
|
expect(profilePromptBlock({}, { waitForCmd: "wf" })).toBe("");
|
|
expect(profilePromptBlock({ ui_settle: "on" }, { waitForCmd: "wf" })).toContain("wf --settle");
|
|
expect(profileLabel({})).toBe("生产默认");
|
|
expect(profileLabel({ effort: "low" })).toBe("思考强度=low");
|
|
});
|
|
});
|
|
|
|
describe("等界面稳定", () => {
|
|
test("Android 指纹命令:按 wm size 跳过状态栏、在设备上算哈希;Override 优先", async () => {
|
|
const { androidScreenHashCmd, sameFingerprint } = await import("@bbbug/shared/settle");
|
|
expect(androidScreenHashCmd("Physical size: 1080x2410\n")).toBe(
|
|
'adb exec-out sh -c "screencap | dd bs=4320 skip=121 count=2169 2>/dev/null | md5sum"');
|
|
expect(androidScreenHashCmd("Physical size: 1440x3200\nOverride size: 1080x2400\n")).toContain("bs=4320 skip=120 count=2160");
|
|
expect(androidScreenHashCmd("error: no devices")).toBeNull();
|
|
expect(sameFingerprint("x", "x")).toBe(true);
|
|
expect(sameFingerprint(null, "x")).toBe(false);
|
|
});
|
|
|
|
test("连续 quiet 不变即稳定;一直在变到超时;探针连续失败报错", async () => {
|
|
const { settleLoop } = await import("@bbbug/shared/settle");
|
|
const fake = () => { let t = 0; return { now: () => t, sleep: async (ms: number) => { t += ms; }, tick: (ms: number) => { t += ms; } }; };
|
|
let c = fake(); let i = 0;
|
|
const seq = ["a", "b", "c", "c", "c", "c", "c", "c", "c"];
|
|
let r = await settleLoop({ probe: async () => { c.tick(100); return seq[Math.min(i++, seq.length - 1)]; }, quietMs: 1000, timeoutMs: 8000, now: c.now, sleep: c.sleep });
|
|
expect(r.settled).toBe(true);
|
|
expect(r.elapsedMs).toBeLessThan(2500);
|
|
c = fake(); i = 0;
|
|
r = await settleLoop({ probe: async () => { c.tick(100); return String(i++); }, quietMs: 1000, timeoutMs: 3000, now: c.now, sleep: c.sleep });
|
|
expect(r.settled).toBe(false);
|
|
expect(r.error).toBeUndefined();
|
|
c = fake();
|
|
r = await settleLoop({ probe: async () => null, quietMs: 1000, timeoutMs: 3000, now: c.now, sleep: c.sleep });
|
|
expect(r.error).toMatch(/取不到/);
|
|
});
|
|
});
|
|
|
|
describe("复跑配对的可比性", () => {
|
|
test("两组跑在不同版本卫星包上的配对不进判定,理由写进结论", () => {
|
|
const T0 = Date.UTC(2026, 9, 5);
|
|
const arm = (wall: number, ui: number) => ({ status: "manual_review", metrics: { ui_wait_s: ui, round_wall_s: wall } });
|
|
const pairs: LabPair[] = [0, 1, 2, 3].map((i) => ({ sourceId: `s${i}`, base: arm(3600, 300), candidate: arm(3500, 100) }));
|
|
pairs[3].incomparable = "两组跑在不同版本的卫星包上(现行 aaaaaaa / 候选 bbbbbbb)";
|
|
const j = judgeLab("ui_wait_s", pairs, T0, T0 + 1);
|
|
expect(j.decision).toBe("pass");
|
|
expect(j.pairsDone).toBe(3);
|
|
pairs[2].incomparable = pairs[3].incomparable;
|
|
const k = judgeLab("ui_wait_s", pairs, T0, T0 + 1);
|
|
expect(k.decision).toBe("fail");
|
|
expect(k.reason).toMatch(/判不了.*不可比/);
|
|
});
|
|
});
|
|
|
|
describe("忙时让路", () => {
|
|
const node = (id: string, fixInFlight: number, clusters = ["default"]) => ({ id, clusters, fixInFlight, fixCapacity: 1 });
|
|
|
|
test("超过阈值才暂停,恰好等于不暂停", () => {
|
|
const five = [node("a", 1), node("b", 1), node("c", 1), node("d", 0), node("e", 0)];
|
|
expect(labLoadGateOf(five, []).paused).toBe(false); // 60% 不算超过
|
|
expect(labLoadGateOf([...five.slice(0, 4), node("e", 1)], []).paused).toBe(true);
|
|
});
|
|
|
|
test("只看 default 集群,在跑的复跑不算负载", () => {
|
|
const nodes = [
|
|
node("lab", 1, ["default", "im-sg"]), node("b", 1), node("c", 0),
|
|
node("temp", 1, ["manual-only"]), node("lemon", 0, ["🍋"]), node("bare", 1, []),
|
|
];
|
|
const g = labLoadGateOf(nodes, ["lab", "temp"]);
|
|
expect(g.capacity).toBe(3);
|
|
expect(g.inFlight).toBe(1);
|
|
expect(g.paused).toBe(false);
|
|
});
|
|
|
|
test("没有在线容量时不暂停", () => {
|
|
expect(labLoadGateOf([], []).paused).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("让路时间不算进复跑期限", () => {
|
|
test("只算落在窗口里的部分,没收口的算到现在", () => {
|
|
const pauses = [{ from: 0, to: 100 }, { from: 150, to: 250 }, { from: 400, to: null }];
|
|
expect(pausedWithin(pauses, 50, 500)).toBe(50 + 100 + 100);
|
|
expect(pausedWithin([], 0, 1000)).toBe(0);
|
|
});
|
|
|
|
test("扣掉让路时间后没到期就继续跑", () => {
|
|
const T0 = Date.UTC(2026, 9, 2);
|
|
const run = { status: null, metrics: {} };
|
|
const pairs: LabPair[] = [{ sourceId: "s", base: run, candidate: run }];
|
|
const now = T0 + 4 * DAY;
|
|
expect(judgeLab("ui_wait_s", pairs, T0, now).decision).toBe("fail");
|
|
expect(judgeLab("ui_wait_s", pairs, T0, now, 2 * DAY).decision).toBe("continue");
|
|
expect(judgeLab("ui_wait_s", pairs, T0, now, 12 * 3_600_000).reason).toMatch(/不含让路给生产的 12\.0 小时/);
|
|
});
|
|
});
|