Files
streamline/satellite/test/eval_timeout_salvage.test.ts
2026-11-16 17:26:44 -08:00

108 lines
5.4 KiB
TypeScript

import { test, expect, describe } from "bun:test";
import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { salvageTimedOutAssessment as salvageBug } from "../src/evaluator";
import { salvageTimedOutAssessment as salvageFeature } from "../src/feature_evaluator";
import { assessSecurityEvalCompletion } from "../src/security_evaluator";
/**
* 生产实测(feature 0f05ce07,fei-android-01,2026-09-08):模型 9.1 分钟就写完了
* `ASSESSMENT: READY` + 闭合的 <PLAN>,但它此前把一条 `find /` 挪到了后台,
* claude -p 进程因此又空转 11 分钟,直到 20 分钟的 EVAL_TIMEOUT_MS 把它杀掉。
* 旧代码在 `timedOut` 分支里无条件丢弃 stdout,于是一份**已经做完**的评估被写成
* 「评估超时,已自动终止」,人得从头再跑一遍。
*
* 这组用例钉住的界限是:**超时只说明进程没退出,不说明没有结论**;
* 但打捞不能变成编造 —— 说了 READY 却没有 plan 的,必须仍然算超时失败,
* 否则下游会拿全是 NEEDS_INPUT 哨兵的默认骨架冒充评估器写的方案。
*/
/** 造一份 `--output-format stream-json` 落盘的样子:末行 result 携带最终正文。 */
function streamJson(finalText: string, opts: { withResult?: boolean } = {}): string {
const lines = [
JSON.stringify({ type: "system", subtype: "init", session_id: "s1" }),
JSON.stringify({ type: "assistant", message: { content: [{ type: "text", text: finalText }] } }),
];
if (opts.withResult !== false) {
lines.push(JSON.stringify({
type: "result", subtype: "error_during_execution", is_error: true,
duration_ms: 1199858, result: finalText,
}));
}
return lines.join("\n") + "\n";
}
const PLAN = "## 目标\n- 确认字段下发\n\n## 判据\n- PASS/FAIL/BLOCKED";
/** Fresh workspaceDir with an optional TestSpec.md already written. */
function makeWorkspace(spec?: string): string {
const dir = mkdtempSync(path.join(tmpdir(), "sec-salvage-"));
if (spec !== undefined) writeFileSync(path.join(dir, "TestSpec.md"), spec, "utf8");
return dir;
}
describe("超时后打捞已产出的评估结论", () => {
test("feature:READY + 完整 <PLAN> —— 采纳,不再当成超时", () => {
const out = streamJson(`ASSESSMENT: READY\nREASON: 目标接口可精确定位\n\n<PLAN>\n${PLAN}\n</PLAN>`);
const a = salvageFeature(out);
expect(a).not.toBeNull();
expect(a!.ready).toBe(true);
expect(a!.plan).toContain("## 目标");
expect(a!.plan).not.toContain("</PLAN>");
});
test("feature:说了 READY 却没有 plan —— 不打捞,宁可报超时也不伪造方案", () => {
expect(salvageFeature(streamJson("ASSESSMENT: READY\nREASON: 就绪"))).toBeNull();
});
test("feature:NEEDS_INFO —— 采纳模型给的原因,并在 reason 里保留「超时」这件事", () => {
const a = salvageFeature(streamJson("ASSESSMENT: NEEDS_INFO\nREASON: 缺少可用账号"));
expect(a).not.toBeNull();
expect(a!.ready).toBe(false);
expect(a!.reason).toContain("缺少可用账号");
expect(a!.reason).toContain("评估超时前已给出结论");
});
test("真卡死、一个字都没写出来 —— 仍然是超时失败", () => {
const stuck = JSON.stringify({ type: "system", subtype: "init", session_id: "s1" }) + "\n";
expect(salvageFeature(stuck)).toBeNull();
expect(salvageBug(stuck)).toBeNull();
});
test("进程被杀得太早、没有末尾 result 行 —— 从 assistant 正文里也要能捞出来", () => {
const out = streamJson(`ASSESSMENT: READY\nREASON: 就绪\n\n<PLAN>\n${PLAN}\n</PLAN>`, { withResult: false });
const a = salvageFeature(out);
expect(a).not.toBeNull();
expect(a!.ready).toBe(true);
expect(a!.plan).toContain("## 判据");
});
test("bug:哨兵是 REPRODUCIBLE,不是 READY", () => {
const a = salvageBug(streamJson(`ASSESSMENT: REPRODUCIBLE\nREASON: 步骤完整\n\n<PLAN>\n${PLAN}\n</PLAN>`));
expect(a).not.toBeNull();
expect(a!.reproducible).toBe(true);
expect(a!.plan).toContain("## 目标");
// READY 是 feature 的词,bug 侧不认。
expect(salvageBug(streamJson(`ASSESSMENT: READY\nREASON: x\n\n<PLAN>\n${PLAN}\n</PLAN>`))).toBeNull();
});
});
test("security:仅按进程退出和本轮 TestSpec 判定,忽略 Agent 的状态文本", () => {
const ws = makeWorkspace(["证据目标", "复现前置", "复现路径", "搜证方法", "判定标准", "风险与前置缺失"]
.map((heading) => `## ${heading}\n- 已确定的搜证路径`).join("\n\n"));
const startedAt = Date.now() - 1_000;
const output = JSON.stringify({ type: "result", subtype: "success", is_error: false,
result: "ASSESSMENT: NEEDS_INFO\nREASON: 实验命中需要运行时确认" });
try {
const completed = assessSecurityEvalCompletion(0, output, ws, startedAt);
expect(completed.ready).toBe(true);
expect(completed.plan).toContain("已确定的搜证路径");
expect(assessSecurityEvalCompletion(1, output, ws, startedAt).ready).toBe(false);
expect(assessSecurityEvalCompletion(0, output, ws, Date.now() + 1_000).ready).toBe(false);
expect(assessSecurityEvalCompletion(0, JSON.stringify({ type: "result", subtype: "success", is_error: true }), ws, startedAt).ready).toBe(false);
writeFileSync(path.join(ws, "TestSpec.md"), " ");
expect(assessSecurityEvalCompletion(0, output, ws, startedAt).ready).toBe(false);
} finally { rmSync(ws, { recursive: true, force: true }); }
});