import { test, expect, describe } from "bun:test"; /** * Dynamic import for the same reason prompt_fixture_cases does it: runner.ts * reads env at module scope. */ const load = async () => (await import("../src/runner")).criticRunFailure; /** One stream-json line, the way the CLI writes it. */ const line = (o: unknown) => JSON.stringify(o) + "\n"; const INIT = line({ type: "system", subtype: "init", session_id: "s1", tools: [], model: "" }); const graded = (verdict: string, risk = 20) => line({ type: "assistant", message: { content: [{ type: "text", text: `SPEC_DEVIATION: NONE\nRISK: ${risk}\nVERDICT: ${verdict}` }] } }) + line({ type: "result", subtype: "success", is_error: false, num_turns: 4, result: `RISK: ${risk}\nVERDICT: ${verdict}` }); describe("criticRunFailure — did anything actually grade this run", () => { test("a graded run is not a failure", async () => { const criticRunFailure = await load(); expect(criticRunFailure(INIT + graded("ROOT_CAUSE"))).toBeNull(); }); test("the real e6613407 crash: error_during_execution on turn 0", async () => { const criticRunFailure = await load(); // Verbatim shape of critic-4.log from the 2026-08-18 incident — the log that // used to be read as "UNCERTAIN · risk 70". const crashed = INIT + line({ type: "result", subtype: "error_during_execution", is_error: true, duration_ms: 120202, num_turns: 0, result: "", stop_reason: null, }); expect(criticRunFailure(crashed)).toBe("error_during_execution、0 turns"); }); test("a killed process that never wrote a result frame", async () => { const criticRunFailure = await load(); expect(criticRunFailure(INIT)).toBe("进程未产出任何 result 帧"); expect(criticRunFailure("")).toBe("进程未产出任何 result 帧"); }); test("a clean exit with no VERDICT line stays a genuine UNCERTAIN", async () => { const criticRunFailure = await load(); // The critic ran, read the fix and hedged without writing the line. That is // an opinion, not a crash, and must NOT be reclassified as ungraded. const rambled = INIT + line({ type: "result", subtype: "success", is_error: false, num_turns: 6, result: "证据只覆盖了正向路径,我无法确定这是不是根因修复。", }); expect(criticRunFailure(rambled)).toBeNull(); }); test("a verdict WINS over a later error frame", async () => { const criticRunFailure = await load(); // It graded the fix and then fell over on the way out. Throwing away a // verdict we actually have would be its own dishonesty. const gradedThenDied = INIT + graded("SCENARIO_BANDAID", 85) + line({ type: "result", subtype: "error_during_execution", is_error: true, num_turns: 0, result: "" }); expect(criticRunFailure(gradedThenDied)).toBeNull(); }); test("the prompt's own VERDICT menu echoed into the log is not a verdict", async () => { const criticRunFailure = await load(); // stdout and stderr share one fd, and the critic prompt lists all four // options verbatim. A CLI that dumps its input on failure must not thereby // look like it graded the fix. const echoed = INIT + "error: request failed. prompt was:\n" + "最后一行四选一:`VERDICT: ROOT_CAUSE` / `VERDICT: SCENARIO_BANDAID` / `VERDICT: INTENT_MISMATCH` / `VERDICT: UNCERTAIN`\n" + line({ type: "result", subtype: "error_during_execution", is_error: true, num_turns: 0, result: "" }); expect(criticRunFailure(echoed)).toBe("error_during_execution、0 turns"); }); test("a retry appended after a crash supersedes it", async () => { const criticRunFailure = await load(); // Both passes live in one critic-.log (the filename grammar in // shared/session_log.ts admits no second file), so the retry's verdict has to // be visible through the crash that precedes it. const crashThenRetry = INIT + line({ type: "result", subtype: "error_during_execution", is_error: true, num_turns: 0, result: "" }) + INIT + graded("ROOT_CAUSE", 15); expect(criticRunFailure(crashThenRetry)).toBeNull(); }); });