import { afterEach, describe, expect, test } from "bun:test"; import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { securityUsageLimitMessage } from "@bbbug/shared/security_agent_failure"; import { artifactFailure, buildSecurityRetryPrompt, classifySecurityAgentResult, securityFailureExtra, validateSecurityArtifacts, } from "../src/security_run_policy"; const dirs: string[] = []; const retryTaskFiles = { workspaceDir: "/tmp/report-task", specPath: "/tmp/report-task/EffectiveSpec.md" }; afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); function result(message: string, isError = true): string { return JSON.stringify({ type: "result", subtype: isError ? "error_during_execution" : "success", is_error: isError, result: message, }); } function validWorkspace(): { dir: string; specPath: string; report: Record } { const dir = mkdtempSync(path.join(tmpdir(), "streamline-security-policy-")); dirs.push(dir); mkdirSync(path.join(dir, "evidence"), { recursive: true }); writeFileSync(path.join(dir, "evidence", "shot-main.png"), "png", "utf8"); const specPath = path.join(dir, "TestSpec.md"); writeFileSync(specPath, `# SecuritySpec ## 判定标准 1. 主路径行为 2. fallback 行为 ## 其他 ... `, "utf8"); writeFileSync(path.join(dir, "EvidenceReport.md"), `# 采证报告 ## 总结结论 已复现。 ## SPEC 验证项逐项对比 | ID | 结论 | 证据 | |---|---|---| | V1 | 已复现 | evidence/shot-main.png | | V2 | 未复现 | 无 | ## 主路径执行记录 已执行。 ## 分支 / fallback / diagnostics 已检查。 ## 已知边界与不可证明项 无。 ## 证据索引 - evidence/shot-main.png `, "utf8"); const report: Record = { overall: "PARTIALLY_REPRODUCED", environment: { platform: "android", device: "device-1", build_ref: "main", account: "test", region: "us", experiments: { exp: "on" }, }, blockers: [], spec_items: [ { id: "V1", point: "主路径行为", verdict: "REPRODUCED", observation: "observed", cause: "expected", evidence: ["evidence/shot-main.png"], }, { id: "V2", point: "fallback 行为", verdict: "NOT_REPRODUCED", observation: "not observed", cause: "condition absent", evidence: [], }, ], main_path: "completed", diagnostics: ["fallback checked"], known_limits: [], evidence_index: ["evidence/shot-main.png"], }; writeFileSync(path.join(dir, "EvidenceReport.json"), `${JSON.stringify(report, null, 2)}\n`, "utf8"); return { dir, specPath, report }; } describe("security provider failure classification", () => { test("API-key usage exhaustion is distinct from transient 429 and successful prose", () => { const previous = process.env.STREAMLINE_CURSOR_AUTH_MODE; process.env.STREAMLINE_CURSOR_AUTH_MODE = "api-key"; try { expect(classifySecurityAgentResult(result('{"code":"usage_limit_exceeded"}'), "cursor")?.code) .toBe("AGENT_USAGE_LIMIT"); expect(classifySecurityAgentResult(result("Service account spend limit reached"), "cursor")?.code) .toBe("AGENT_USAGE_LIMIT"); expect(classifySecurityAgentResult(result("You have reached your team's spending limit"), "cursor")?.code) .toBe("AGENT_USAGE_LIMIT"); expect(classifySecurityAgentResult(result("HTTP 429 too many requests"), "cursor")?.code) .toBe("AGENT_RATE_LIMIT"); expect(classifySecurityAgentResult(result("The test discusses usage limit exceeded", false), "cursor")) .toBeNull(); } finally { if (previous === undefined) delete process.env.STREAMLINE_CURSOR_AUTH_MODE; else process.env.STREAMLINE_CURSOR_AUTH_MODE = previous; } }); test("recognises Claude, Codex and Cursor usage exhaustion", () => { expect(classifySecurityAgentResult(result("You've hit your limit · resets at 5pm"), "claude")?.code) .toBe("AGENT_USAGE_LIMIT"); expect(classifySecurityAgentResult(result("ERROR: You've hit your usage limit. Try again at 8:00 PM."), "codex")?.code) .toBe("AGENT_USAGE_LIMIT"); expect(classifySecurityAgentResult(result("You've hit your usage limit. Switch to Auto for more usage."), "cursor")?.code) .toBe("AGENT_USAGE_LIMIT"); expect(classifySecurityAgentResult(result("You've reached your monthly usage limit Request higher limits to continue using Cursor"), "cursor")?.code) .toBe("AGENT_USAGE_LIMIT"); expect(securityUsageLimitMessage("claude", "\u001b[31mYou've hit your limit · resets at 5pm\u001b[0m")) .toContain("You've hit your limit"); }); test("recognises Claude's structured rejected rate-limit event", () => { const stream = JSON.stringify({ type: "rate_limit_event", rate_limit_info: { status: "rejected", rateLimitType: "five_hour", resetsAt: 1_800_000_000, overageDisabledReason: "org_level_disabled_until", }, }); expect(classifySecurityAgentResult(stream, "claude")).toMatchObject({ code: "AGENT_USAGE_LIMIT", reason: expect.stringContaining("rateLimitType=five_hour"), }); }); test("keeps rate, context, gateway and incomplete failures distinct", () => { expect(classifySecurityAgentResult(result("HTTP 429 too many requests"), "claude")?.code) .toBe("AGENT_RATE_LIMIT"); expect(classifySecurityAgentResult(result("maximum context length exceeded"), "codex")?.code) .toBe("AGENT_CONTEXT_LIMIT"); expect(classifySecurityAgentResult('{"type":"error","message":"service unavailable HTTP 503"}', "cursor")?.code) .toBe("AGENT_GATEWAY_ERROR"); expect(classifySecurityAgentResult("", "claude")?.code) .toBe("AGENT_STREAM_INCOMPLETE"); }); test("does not classify ordinary agent prose as a CLI quota failure", () => { const stream = [ JSON.stringify({ type: "assistant", message: { model: "claude-sonnet", content: [{ type: "text", text: "The app displays a usage limit dialog." }] }, }), result("reports written", false), ].join("\n"); expect(classifySecurityAgentResult(stream, "claude")).toBeNull(); }); test("keeps the concrete quota reason for callbacks without replaying the banner into the PTY", () => { const failure = classifySecurityAgentResult( result("You've hit your usage limit. Try again at 8:00 PM."), "codex", )!; expect(buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, "en")).not.toContain("You've hit your usage limit"); expect(securityFailureExtra(failure, 3, 3).failure_reason).toContain("You've hit your usage limit"); }); }); describe("security evidence artifact validation", () => { test("accepts two consistent, schema-valid reports and their evidence", () => { const fixture = validWorkspace(); expect(validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath })) .toMatchObject({ ok: true, errors: [] }); }); test("rejects a missing JSON report", () => { const fixture = validWorkspace(); unlinkSync(path.join(fixture.dir, "EvidenceReport.json")); const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }); expect(validation.ok).toBe(false); expect(validation.errors.some((error) => error.code === "JSON_MISSING")).toBe(true); }); test("rejects malformed JSON, missing spec rows and missing evidence files", () => { const malformed = validWorkspace(); writeFileSync(path.join(malformed.dir, "EvidenceReport.json"), "{", "utf8"); expect(validateSecurityArtifacts({ workspaceDir: malformed.dir, specPath: malformed.specPath }).errors .some((error) => error.code === "JSON_PARSE")).toBe(true); const mismatch = validWorkspace(); const report = structuredClone(mismatch.report) as Record; report.spec_items = (report.spec_items as Array>).slice(0, 1); writeFileSync(path.join(mismatch.dir, "EvidenceReport.json"), JSON.stringify(report), "utf8"); unlinkSync(path.join(mismatch.dir, "evidence", "shot-main.png")); const validation = validateSecurityArtifacts({ workspaceDir: mismatch.dir, specPath: mismatch.specPath }); expect(validation.errors.some((error) => error.code === "SPEC_ITEMS_MISSING")).toBe(true); expect(validation.errors.some((error) => error.code === "EVIDENCE_INVALID")).toBe(true); }); test("builds a report repair prompt and structured final reason", () => { const fixture = validWorkspace(); unlinkSync(path.join(fixture.dir, "EvidenceReport.json")); const failure = artifactFailure( validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }), "cursor", ); const prompt = buildSecurityRetryPrompt(failure, 2, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, "zh"); expect(prompt).toContain("上一轮报告缺失或未满足下面的要求"); expect(prompt).toContain("JSON_MISSING"); expect(prompt).toContain("保存文件后结束,无需输出额外状态标记"); expect(securityFailureExtra(failure, 3, 3)).toMatchObject({ failure_code: "ARTIFACT_INVALID", provider: "cursor", retry_count: 3, max_retries: 3, }); }); for (const lang of ["zh", "en"] as const) { test(`missing reports receive the full format and task-specific IDs (${lang})`, () => { const fixture = validWorkspace(); writeFileSync(fixture.specPath, "## 判定标准\n- V3: 主路径\n- V7: 回退路径\n", "utf8"); unlinkSync(path.join(fixture.dir, "EvidenceReport.md")); unlinkSync(path.join(fixture.dir, "EvidenceReport.json")); const failure = artifactFailure( validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }), "cursor", ); expect(failure.expectedSpecIds).toEqual(["V3", "V7"]); const prompt = buildSecurityRetryPrompt(failure, 1, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, lang); expect(prompt).toContain("V3, V7"); expect(prompt).toContain("MD_MISSING [EvidenceReport.md @"); expect(prompt).toContain("JSON_MISSING [EvidenceReport.json @"); expect(prompt).toContain(lang === "en" ? "## Known limits" : "## 已知边界与不可证明项"); expect(prompt).toContain(lang === "en" ? "copy that real file into evidence/" : "先把真实文件复制到 evidence/"); const example = JSON.parse(prompt.match(/```json\n([\s\S]*?)\n```/)![1]!); expect(example).toMatchObject({ overall: "EVIDENCE_INSUFFICIENT", environment: { platform: "", experiments: expect.any(Object) }, blockers: [], spec_items: [{ id: "V3", evidence: [], observation: expect.any(String), cause: expect.any(String) }], main_path: expect.any(String), diagnostics: [], known_limits: [], evidence_index: [], }); }); } test("locates invalid IDs, nested fields and every reference to an invalid evidence path", () => { const fixture = validWorkspace(); const report = structuredClone(fixture.report); (report.environment as Record).device = 123; const items = report.spec_items as Array>; items[0]!.id = "VP-P-01"; items[0]!.verdict = "PASS"; items[0]!.evidence = ["artifacts/probe.js"]; items[1]!.evidence = [7]; report.evidence_index = ["artifacts/probe.js"]; writeFileSync(path.join(fixture.dir, "EvidenceReport.json"), JSON.stringify(report), "utf8"); const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }); expect(validation.errors).toEqual(expect.arrayContaining([ expect.objectContaining({ code: "JSON_SCHEMA", location: "$.environment.device" }), expect.objectContaining({ code: "SPEC_ID_FORMAT", location: "$.spec_items[0].id" }), expect.objectContaining({ code: "JSON_ENUM", location: "$.spec_items[0].verdict" }), expect.objectContaining({ code: "JSON_SCHEMA", location: "$.spec_items[1].evidence[0]" }), expect.objectContaining({ code: "SPEC_ITEMS_MISSING", location: "$.spec_items[*].id" }), expect.objectContaining({ code: "EVIDENCE_INVALID", location: "EvidenceReport.json: $.evidence_index[0], $.spec_items[0].evidence[0]", }), expect.objectContaining({ code: "MD_EVIDENCE_MISSING", location: "## 证据索引 / Evidence index" }), ])); const failure = artifactFailure(validation, "cursor"); for (const lang of ["zh", "en"] as const) { const prompt = buildSecurityRetryPrompt(failure, 2, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, lang); expect(prompt).toContain("SPEC_ID_FORMAT [EvidenceReport.json @ $.spec_items[0].id]"); expect(prompt).toContain("VP-P-01"); expect(prompt).toContain("$.evidence_index[0], $.spec_items[0].evidence[0]"); expect(prompt).toContain("artifacts/probe.js"); } }); test("locates malformed JSON at the document root and missing Markdown sections by heading", () => { const fixture = validWorkspace(); writeFileSync(path.join(fixture.dir, "EvidenceReport.json"), "{", "utf8"); writeFileSync(path.join(fixture.dir, "EvidenceReport.md"), "# 采证报告\n## 总结结论\n待补齐。", "utf8"); const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }); expect(validation.errors).toEqual(expect.arrayContaining([ expect.objectContaining({ code: "JSON_PARSE", file: "EvidenceReport.json", location: "$" }), expect.objectContaining({ code: "MD_SECTION_MISSING", file: "EvidenceReport.md", location: "## 主路径执行记录 / Main path" }), ])); }); test("continuation after resource exhaustion preserves every required report item", () => { const fixture = validWorkspace(); writeFileSync(fixture.specPath, `## 判定标准\n${Array.from({ length: 9 }, (_, i) => `${i + 1}. 规则 ${i + 1}`).join("\n")}\n`, "utf8"); const failure = classifySecurityAgentResult(result("RetriableError: [resource_exhausted] Error"), "cursor")!; const files = { workspaceDir: fixture.dir, specPath: fixture.specPath }; const zh = buildSecurityRetryPrompt(failure, 3, 3, files, "zh"); const en = buildSecurityRetryPrompt(failure, 3, 3, files, "en"); expect(zh).toContain("资源耗尽"); expect(en).toContain("resource exhaustion"); for (const id of ["V1", "V7", "V8", "V9"]) { expect(zh).toContain(id); expect(en).toContain(id); } expect(zh).toContain("前置阻塞或证据不足也要保留该项"); expect(zh).not.toContain("具体错误及位置"); }); test("does not attach the report schema to provider failures", () => { const failure = classifySecurityAgentResult(result("HTTP 503 service unavailable"), "cursor")!; expect(failure.code).toBe("AGENT_GATEWAY_ERROR"); for (const lang of ["zh", "en"] as const) { const prompt = buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, lang); expect(prompt).not.toContain("```json"); expect(prompt).toContain(lang === "en" ? "previous attempt was interrupted" : "上一次执行中断"); expect(prompt).not.toContain(lang === "en" ? "output format is incorrect" : "输出格式不对"); } }); test("repair prompts name the actual task and output paths without platform internals", () => { const fixture = validWorkspace(); const activeSpecPath = path.join(fixture.dir, "EffectiveSpec.md"); writeFileSync(activeSpecPath, "## 判定标准\n- V4: 修正后的判定标准\n", "utf8"); unlinkSync(path.join(fixture.dir, "EvidenceReport.json")); const failure = artifactFailure( validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: activeSpecPath }), "cursor", ); for (const lang of ["zh", "en"] as const) { // These paths intentionally differ from cwd and use the active spec's // filename; the prompt must not send the agent back to TestSpec.md. const prompt = buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, lang); expect(prompt).toContain("/tmp/report-task/EffectiveSpec.md"); expect(prompt).toContain("/tmp/report-task/EvidenceReport.md"); expect(prompt).toContain("/tmp/report-task/EvidenceReport.json"); expect(prompt).toContain("/tmp/report-task/evidence"); expect(prompt).toContain("V4"); expect(prompt).not.toContain("TestSpec"); expect(prompt).not.toMatch(/\b(?:security|streamline|satellite|runner|provider|validator)\b|post-run hook|卫星机|原始契约|校验器/i); expect(prompt).not.toContain("EVIDENCE_READY"); expect(prompt).not.toContain("ARTIFACT_INVALID"); expect(prompt).toContain(lang === "en" ? "Read the task file without changing its requirements" : "读取任务说明,不要修改其中的任务要求"); } }); });