Files
streamline/satellite/test/experiments.test.ts
T
2026-11-16 17:26:44 -08:00

117 lines
5.7 KiB
TypeScript

/**
* 按节点关改动的开关 —— 给效果归因留对照组用。
*
* 2026-09-17 想查 wait-for 上线后单轮时长为什么涨了 47%,发现没有对照组:以为其它
* 节点池还跑旧 bundle,实际那几天已经升到同一版。于是既不能归因也不能排除。
* 直接补救(部署时漏掉一台)的代价是那台机器同时错过其它所有改动,而且越拖越不干净。
*
* 这里测的是两件事:**默认必须全开**(拼错的开关名不该悄悄关掉生产行为),
* 以及**被关掉的那几项要能上报**(否则回测时又不知道哪台跑的是哪种行为)。
*/
import { describe, expect, test } from "bun:test";
import {
parseExperiments, experimentOff, experimentOverrides, EXPERIMENT_NAMES, EXPERIMENT_NOTES,
} from "@bbbug/shared/experiments";
describe("实验开关", () => {
test("没配 = 全开", () => {
expect(parseExperiments(undefined)).toEqual({});
expect(experimentOff("waitfor", undefined)).toBe(false);
expect(experimentOff("waitfor", "")).toBe(false);
expect(experimentOverrides("")).toEqual([]);
});
test("显式 off 才关", () => {
expect(experimentOff("waitfor", "waitfor=off")).toBe(true);
expect(experimentOff("waitfor", "waitfor=OFF")).toBe(true);
expect(experimentOff("waitfor", " waitfor = off ")).toBe(true);
expect(experimentOff("waitfor", "waitfor=on")).toBe(false);
});
test("写错的值一律当作开 —— 不能把生产行为悄悄关掉", () => {
for (const bad of ["waitfor=", "waitfor=0", "waitfor=false", "waitfor=disabled", "waitfor"]) {
expect(experimentOff("waitfor", bad)).toBe(false);
}
});
test("不认识的开关名忽略,不阻断,也不影响别的项", () => {
expect(parseExperiments("nosuchflag=off")).toEqual({});
expect(experimentOff("waitfor", "nosuchflag=off,waitfor=off")).toBe(true);
});
test("多项、空段、乱空格都能解析", () => {
expect(experimentOverrides("waitfor=off,, ,nosuch=off")).toEqual(["waitfor=off"]);
});
test("上报的只有被改过的那几项 —— 全开时是空数组,不是全表", () => {
expect(experimentOverrides("")).toEqual([]);
expect(experimentOverrides("waitfor=off")).toEqual(["waitfor=off"]);
});
test("每个开关都得写明「关掉之后是什么行为」", () => {
for (const n of EXPERIMENT_NAMES) {
expect(EXPERIMENT_NOTES[n]).toBeTruthy();
expect(EXPERIMENT_NOTES[n].length).toBeGreaterThan(20);
}
});
});
describe("开关真的关掉了提示词那一节", () => {
test("waitfor=off 时 waitPromptBlock 返回空串,且 ensure* 跟着 no-op", async () => {
const prev = process.env.STREAMLINE_EXPERIMENTS;
try {
process.env.STREAMLINE_EXPERIMENTS = "waitfor=off";
const { waitPromptBlock, ensureWaitBlockInPrompt } = await import("@bbbug/shared/wait_prompt");
const block = waitPromptBlock({ cmd: "/x/satellite.js wait-for", lang: "zh" });
expect(block).toBe("");
// 空串进 ensure* 必须原样返回 —— 这是「一处改、四个 runner 全覆盖」的依据
const p = "## 步骤 3 — 复现、修复、验证\n正文\n";
expect(ensureWaitBlockInPrompt(p, block)).toBe(p);
} finally {
if (prev == null) delete process.env.STREAMLINE_EXPERIMENTS;
else process.env.STREAMLINE_EXPERIMENTS = prev;
}
});
});
describe("autocompact:启动 agent 时带上自动压缩窗口", () => {
test("默认开:设 300k;节点可改值;off 时不设(对照组退回改动前)", async () => {
const { agentSpawnEnv, AUTO_COMPACT_WINDOW } = await import("../src/agent_env");
const saved = { e: process.env.STREAMLINE_EXPERIMENTS, w: process.env.STREAMLINE_AUTO_COMPACT_WINDOW, c: process.env.CLAUDE_CODE_AUTO_COMPACT_WINDOW };
try {
delete process.env.CLAUDE_CODE_AUTO_COMPACT_WINDOW;
delete process.env.STREAMLINE_EXPERIMENTS; delete process.env.STREAMLINE_AUTO_COMPACT_WINDOW;
expect((await agentSpawnEnv()).CLAUDE_CODE_AUTO_COMPACT_WINDOW).toBe(String(AUTO_COMPACT_WINDOW));
process.env.STREAMLINE_AUTO_COMPACT_WINDOW = "400000";
expect((await agentSpawnEnv()).CLAUDE_CODE_AUTO_COMPACT_WINDOW).toBe("400000");
process.env.STREAMLINE_EXPERIMENTS = "autocompact=off";
expect((await agentSpawnEnv()).CLAUDE_CODE_AUTO_COMPACT_WINDOW).toBeUndefined();
expect(experimentOverrides("autocompact=off,lightshots=off")).toEqual(["autocompact=off", "lightshots=off"]);
} finally {
for (const [k, v] of [["STREAMLINE_EXPERIMENTS", saved.e], ["STREAMLINE_AUTO_COMPACT_WINDOW", saved.w], ["CLAUDE_CODE_AUTO_COMPACT_WINDOW", saved.c]] as const) {
if (v === undefined) delete process.env[k]; else process.env[k] = v;
}
}
});
});
describe("lightshots:修 bug / Feature 的看图节制", () => {
test("开着:插在收尾前,已有就不再插;关掉:不插", async () => {
const { lightShotsBlock, ensureLightShotsInPrompt } = await import("@bbbug/shared/shots_prompt");
const saved = process.env.STREAMLINE_EXPERIMENTS;
try {
delete process.env.STREAMLINE_EXPERIMENTS;
const p = "## 步骤 3 — 修复\n…\n## 收尾\n打印 STATUS";
const once = ensureLightShotsInPrompt(p, lightShotsBlock("zh"));
expect(once.indexOf("## 看图节制")).toBeLessThan(once.indexOf("## 收尾"));
expect(once).toContain("sips -Z 480");
expect(ensureLightShotsInPrompt(once, lightShotsBlock("zh"))).toBe(once);
expect(lightShotsBlock("en")).toContain("Go easy on screenshots");
process.env.STREAMLINE_EXPERIMENTS = "lightshots=off";
expect(ensureLightShotsInPrompt(p, lightShotsBlock("zh"))).toBe(p);
} finally {
if (saved === undefined) delete process.env.STREAMLINE_EXPERIMENTS; else process.env.STREAMLINE_EXPERIMENTS = saved;
}
});
});