import { test, expect, describe } from "bun:test"; import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { salvageTimedOutAssessment as salvageBug } from "../src/evaluator"; import { salvageTimedOutAssessment as salvageFeature } from "../src/feature_evaluator"; import { assessSecurityEvalCompletion } from "../src/security_evaluator"; /** * 生产实测(feature 0f05ce07,fei-android-01,2026-09-08):模型 9.1 分钟就写完了 * `ASSESSMENT: READY` + 闭合的 ,但它此前把一条 `find /` 挪到了后台, * claude -p 进程因此又空转 11 分钟,直到 20 分钟的 EVAL_TIMEOUT_MS 把它杀掉。 * 旧代码在 `timedOut` 分支里无条件丢弃 stdout,于是一份**已经做完**的评估被写成 * 「评估超时,已自动终止」,人得从头再跑一遍。 * * 这组用例钉住的界限是:**超时只说明进程没退出,不说明没有结论**; * 但打捞不能变成编造 —— 说了 READY 却没有 plan 的,必须仍然算超时失败, * 否则下游会拿全是 NEEDS_INPUT 哨兵的默认骨架冒充评估器写的方案。 */ /** 造一份 `--output-format stream-json` 落盘的样子:末行 result 携带最终正文。 */ function streamJson(finalText: string, opts: { withResult?: boolean } = {}): string { const lines = [ JSON.stringify({ type: "system", subtype: "init", session_id: "s1" }), JSON.stringify({ type: "assistant", message: { content: [{ type: "text", text: finalText }] } }), ]; if (opts.withResult !== false) { lines.push(JSON.stringify({ type: "result", subtype: "error_during_execution", is_error: true, duration_ms: 1199858, result: finalText, })); } return lines.join("\n") + "\n"; } const PLAN = "## 目标\n- 确认字段下发\n\n## 判据\n- PASS/FAIL/BLOCKED"; /** Fresh workspaceDir with an optional TestSpec.md already written. */ function makeWorkspace(spec?: string): string { const dir = mkdtempSync(path.join(tmpdir(), "sec-salvage-")); if (spec !== undefined) writeFileSync(path.join(dir, "TestSpec.md"), spec, "utf8"); return dir; } describe("超时后打捞已产出的评估结论", () => { test("feature:READY + 完整 —— 采纳,不再当成超时", () => { const out = streamJson(`ASSESSMENT: READY\nREASON: 目标接口可精确定位\n\n\n${PLAN}\n`); const a = salvageFeature(out); expect(a).not.toBeNull(); expect(a!.ready).toBe(true); expect(a!.plan).toContain("## 目标"); expect(a!.plan).not.toContain(""); }); test("feature:说了 READY 却没有 plan —— 不打捞,宁可报超时也不伪造方案", () => { expect(salvageFeature(streamJson("ASSESSMENT: READY\nREASON: 就绪"))).toBeNull(); }); test("feature:NEEDS_INFO —— 采纳模型给的原因,并在 reason 里保留「超时」这件事", () => { const a = salvageFeature(streamJson("ASSESSMENT: NEEDS_INFO\nREASON: 缺少可用账号")); expect(a).not.toBeNull(); expect(a!.ready).toBe(false); expect(a!.reason).toContain("缺少可用账号"); expect(a!.reason).toContain("评估超时前已给出结论"); }); test("真卡死、一个字都没写出来 —— 仍然是超时失败", () => { const stuck = JSON.stringify({ type: "system", subtype: "init", session_id: "s1" }) + "\n"; expect(salvageFeature(stuck)).toBeNull(); expect(salvageBug(stuck)).toBeNull(); }); test("进程被杀得太早、没有末尾 result 行 —— 从 assistant 正文里也要能捞出来", () => { const out = streamJson(`ASSESSMENT: READY\nREASON: 就绪\n\n\n${PLAN}\n`, { withResult: false }); const a = salvageFeature(out); expect(a).not.toBeNull(); expect(a!.ready).toBe(true); expect(a!.plan).toContain("## 判据"); }); test("bug:哨兵是 REPRODUCIBLE,不是 READY", () => { const a = salvageBug(streamJson(`ASSESSMENT: REPRODUCIBLE\nREASON: 步骤完整\n\n\n${PLAN}\n`)); expect(a).not.toBeNull(); expect(a!.reproducible).toBe(true); expect(a!.plan).toContain("## 目标"); // READY 是 feature 的词,bug 侧不认。 expect(salvageBug(streamJson(`ASSESSMENT: READY\nREASON: x\n\n\n${PLAN}\n`))).toBeNull(); }); }); test("security:仅按进程退出和本轮 TestSpec 判定,忽略 Agent 的状态文本", () => { const ws = makeWorkspace(["证据目标", "复现前置", "复现路径", "搜证方法", "判定标准", "风险与前置缺失"] .map((heading) => `## ${heading}\n- 已确定的搜证路径`).join("\n\n")); const startedAt = Date.now() - 1_000; const output = JSON.stringify({ type: "result", subtype: "success", is_error: false, result: "ASSESSMENT: NEEDS_INFO\nREASON: 实验命中需要运行时确认" }); try { const completed = assessSecurityEvalCompletion(0, output, ws, startedAt); expect(completed.ready).toBe(true); expect(completed.plan).toContain("已确定的搜证路径"); expect(assessSecurityEvalCompletion(1, output, ws, startedAt).ready).toBe(false); expect(assessSecurityEvalCompletion(0, output, ws, Date.now() + 1_000).ready).toBe(false); expect(assessSecurityEvalCompletion(0, JSON.stringify({ type: "result", subtype: "success", is_error: true }), ws, startedAt).ready).toBe(false); writeFileSync(path.join(ws, "TestSpec.md"), " "); expect(assessSecurityEvalCompletion(0, output, ws, startedAt).ready).toBe(false); } finally { rmSync(ws, { recursive: true, force: true }); } });