Files
streamline/satellite/test/critic_run_failure.test.ts
T
2026-11-16 17:26:44 -08:00

83 lines
4.1 KiB
TypeScript

import { test, expect, describe } from "bun:test";
/**
* Dynamic import for the same reason prompt_fixture_cases does it: runner.ts
* reads env at module scope.
*/
const load = async () => (await import("../src/runner")).criticRunFailure;
/** One stream-json line, the way the CLI writes it. */
const line = (o: unknown) => JSON.stringify(o) + "\n";
const INIT = line({ type: "system", subtype: "init", session_id: "s1", tools: [], model: "" });
const graded = (verdict: string, risk = 20) =>
line({ type: "assistant", message: { content: [{ type: "text", text: `SPEC_DEVIATION: NONE\nRISK: ${risk}\nVERDICT: ${verdict}` }] } })
+ line({ type: "result", subtype: "success", is_error: false, num_turns: 4, result: `RISK: ${risk}\nVERDICT: ${verdict}` });
describe("criticRunFailure — did anything actually grade this run", () => {
test("a graded run is not a failure", async () => {
const criticRunFailure = await load();
expect(criticRunFailure(INIT + graded("ROOT_CAUSE"))).toBeNull();
});
test("the real e6613407 crash: error_during_execution on turn 0", async () => {
const criticRunFailure = await load();
// Verbatim shape of critic-4.log from the 2026-08-18 incident — the log that
// used to be read as "UNCERTAIN · risk 70".
const crashed = INIT + line({
type: "result", subtype: "error_during_execution", is_error: true,
duration_ms: 120202, num_turns: 0, result: "", stop_reason: null,
});
expect(criticRunFailure(crashed)).toBe("error_during_execution、0 turns");
});
test("a killed process that never wrote a result frame", async () => {
const criticRunFailure = await load();
expect(criticRunFailure(INIT)).toBe("进程未产出任何 result 帧");
expect(criticRunFailure("")).toBe("进程未产出任何 result 帧");
});
test("a clean exit with no VERDICT line stays a genuine UNCERTAIN", async () => {
const criticRunFailure = await load();
// The critic ran, read the fix and hedged without writing the line. That is
// an opinion, not a crash, and must NOT be reclassified as ungraded.
const rambled = INIT + line({
type: "result", subtype: "success", is_error: false, num_turns: 6,
result: "证据只覆盖了正向路径,我无法确定这是不是根因修复。",
});
expect(criticRunFailure(rambled)).toBeNull();
});
test("a verdict WINS over a later error frame", async () => {
const criticRunFailure = await load();
// It graded the fix and then fell over on the way out. Throwing away a
// verdict we actually have would be its own dishonesty.
const gradedThenDied = INIT + graded("SCENARIO_BANDAID", 85)
+ line({ type: "result", subtype: "error_during_execution", is_error: true, num_turns: 0, result: "" });
expect(criticRunFailure(gradedThenDied)).toBeNull();
});
test("the prompt's own VERDICT menu echoed into the log is not a verdict", async () => {
const criticRunFailure = await load();
// stdout and stderr share one fd, and the critic prompt lists all four
// options verbatim. A CLI that dumps its input on failure must not thereby
// look like it graded the fix.
const echoed = INIT
+ "error: request failed. prompt was:\n"
+ "最后一行四选一:`VERDICT: ROOT_CAUSE` / `VERDICT: SCENARIO_BANDAID` / `VERDICT: INTENT_MISMATCH` / `VERDICT: UNCERTAIN`\n"
+ line({ type: "result", subtype: "error_during_execution", is_error: true, num_turns: 0, result: "" });
expect(criticRunFailure(echoed)).toBe("error_during_execution、0 turns");
});
test("a retry appended after a crash supersedes it", async () => {
const criticRunFailure = await load();
// Both passes live in one critic-<round>.log (the filename grammar in
// shared/session_log.ts admits no second file), so the retry's verdict has to
// be visible through the crash that precedes it.
const crashThenRetry =
INIT + line({ type: "result", subtype: "error_during_execution", is_error: true, num_turns: 0, result: "" })
+ INIT + graded("ROOT_CAUSE", 15);
expect(criticRunFailure(crashThenRetry)).toBeNull();
});
});