Initial import of streamline
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
import { test, expect, describe } from "bun:test";
|
||||
|
||||
/**
|
||||
* Dynamic import for the same reason prompt_fixture_cases does it: runner.ts
|
||||
* reads env at module scope.
|
||||
*/
|
||||
const load = async () => (await import("../src/runner")).criticRunFailure;
|
||||
|
||||
/** One stream-json line, the way the CLI writes it. */
|
||||
const line = (o: unknown) => JSON.stringify(o) + "\n";
|
||||
|
||||
const INIT = line({ type: "system", subtype: "init", session_id: "s1", tools: [], model: "" });
|
||||
const graded = (verdict: string, risk = 20) =>
|
||||
line({ type: "assistant", message: { content: [{ type: "text", text: `SPEC_DEVIATION: NONE\nRISK: ${risk}\nVERDICT: ${verdict}` }] } })
|
||||
+ line({ type: "result", subtype: "success", is_error: false, num_turns: 4, result: `RISK: ${risk}\nVERDICT: ${verdict}` });
|
||||
|
||||
describe("criticRunFailure — did anything actually grade this run", () => {
|
||||
test("a graded run is not a failure", async () => {
|
||||
const criticRunFailure = await load();
|
||||
expect(criticRunFailure(INIT + graded("ROOT_CAUSE"))).toBeNull();
|
||||
});
|
||||
|
||||
test("the real e6613407 crash: error_during_execution on turn 0", async () => {
|
||||
const criticRunFailure = await load();
|
||||
// Verbatim shape of critic-4.log from the 2026-08-18 incident — the log that
|
||||
// used to be read as "UNCERTAIN · risk 70".
|
||||
const crashed = INIT + line({
|
||||
type: "result", subtype: "error_during_execution", is_error: true,
|
||||
duration_ms: 120202, num_turns: 0, result: "", stop_reason: null,
|
||||
});
|
||||
expect(criticRunFailure(crashed)).toBe("error_during_execution、0 turns");
|
||||
});
|
||||
|
||||
test("a killed process that never wrote a result frame", async () => {
|
||||
const criticRunFailure = await load();
|
||||
expect(criticRunFailure(INIT)).toBe("进程未产出任何 result 帧");
|
||||
expect(criticRunFailure("")).toBe("进程未产出任何 result 帧");
|
||||
});
|
||||
|
||||
test("a clean exit with no VERDICT line stays a genuine UNCERTAIN", async () => {
|
||||
const criticRunFailure = await load();
|
||||
// The critic ran, read the fix and hedged without writing the line. That is
|
||||
// an opinion, not a crash, and must NOT be reclassified as ungraded.
|
||||
const rambled = INIT + line({
|
||||
type: "result", subtype: "success", is_error: false, num_turns: 6,
|
||||
result: "证据只覆盖了正向路径,我无法确定这是不是根因修复。",
|
||||
});
|
||||
expect(criticRunFailure(rambled)).toBeNull();
|
||||
});
|
||||
|
||||
test("a verdict WINS over a later error frame", async () => {
|
||||
const criticRunFailure = await load();
|
||||
// It graded the fix and then fell over on the way out. Throwing away a
|
||||
// verdict we actually have would be its own dishonesty.
|
||||
const gradedThenDied = INIT + graded("SCENARIO_BANDAID", 85)
|
||||
+ line({ type: "result", subtype: "error_during_execution", is_error: true, num_turns: 0, result: "" });
|
||||
expect(criticRunFailure(gradedThenDied)).toBeNull();
|
||||
});
|
||||
|
||||
test("the prompt's own VERDICT menu echoed into the log is not a verdict", async () => {
|
||||
const criticRunFailure = await load();
|
||||
// stdout and stderr share one fd, and the critic prompt lists all four
|
||||
// options verbatim. A CLI that dumps its input on failure must not thereby
|
||||
// look like it graded the fix.
|
||||
const echoed = INIT
|
||||
+ "error: request failed. prompt was:\n"
|
||||
+ "最后一行四选一:`VERDICT: ROOT_CAUSE` / `VERDICT: SCENARIO_BANDAID` / `VERDICT: INTENT_MISMATCH` / `VERDICT: UNCERTAIN`\n"
|
||||
+ line({ type: "result", subtype: "error_during_execution", is_error: true, num_turns: 0, result: "" });
|
||||
expect(criticRunFailure(echoed)).toBe("error_during_execution、0 turns");
|
||||
});
|
||||
|
||||
test("a retry appended after a crash supersedes it", async () => {
|
||||
const criticRunFailure = await load();
|
||||
// Both passes live in one critic-<round>.log (the filename grammar in
|
||||
// shared/session_log.ts admits no second file), so the retry's verdict has to
|
||||
// be visible through the crash that precedes it.
|
||||
const crashThenRetry =
|
||||
INIT + line({ type: "result", subtype: "error_during_execution", is_error: true, num_turns: 0, result: "" })
|
||||
+ INIT + graded("ROOT_CAUSE", 15);
|
||||
expect(criticRunFailure(crashThenRetry)).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user