357 lines
17 KiB
TypeScript
357 lines
17 KiB
TypeScript
import { afterEach, describe, expect, test } from "bun:test";
|
|
import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { securityUsageLimitMessage } from "@bbbug/shared/security_agent_failure";
|
|
import {
|
|
artifactFailure,
|
|
buildSecurityRetryPrompt,
|
|
classifySecurityAgentResult,
|
|
securityFailureExtra,
|
|
validateSecurityArtifacts,
|
|
} from "../src/security_run_policy";
|
|
|
|
const dirs: string[] = [];
|
|
const retryTaskFiles = { workspaceDir: "/tmp/report-task", specPath: "/tmp/report-task/EffectiveSpec.md" };
|
|
|
|
afterEach(() => {
|
|
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
function result(message: string, isError = true): string {
|
|
return JSON.stringify({
|
|
type: "result",
|
|
subtype: isError ? "error_during_execution" : "success",
|
|
is_error: isError,
|
|
result: message,
|
|
});
|
|
}
|
|
|
|
function validWorkspace(): { dir: string; specPath: string; report: Record<string, unknown> } {
|
|
const dir = mkdtempSync(path.join(tmpdir(), "streamline-security-policy-"));
|
|
dirs.push(dir);
|
|
mkdirSync(path.join(dir, "evidence"), { recursive: true });
|
|
writeFileSync(path.join(dir, "evidence", "shot-main.png"), "png", "utf8");
|
|
const specPath = path.join(dir, "TestSpec.md");
|
|
writeFileSync(specPath, `# SecuritySpec
|
|
|
|
## 判定标准
|
|
1. 主路径行为
|
|
2. fallback 行为
|
|
|
|
## 其他
|
|
...
|
|
`, "utf8");
|
|
writeFileSync(path.join(dir, "EvidenceReport.md"), `# 采证报告
|
|
|
|
## 总结结论
|
|
已复现。
|
|
|
|
## SPEC 验证项逐项对比
|
|
| ID | 结论 | 证据 |
|
|
|---|---|---|
|
|
| V1 | 已复现 | evidence/shot-main.png |
|
|
| V2 | 未复现 | 无 |
|
|
|
|
## 主路径执行记录
|
|
已执行。
|
|
|
|
## 分支 / fallback / diagnostics
|
|
已检查。
|
|
|
|
## 已知边界与不可证明项
|
|
无。
|
|
|
|
## 证据索引
|
|
- evidence/shot-main.png
|
|
`, "utf8");
|
|
const report: Record<string, unknown> = {
|
|
overall: "PARTIALLY_REPRODUCED",
|
|
environment: {
|
|
platform: "android",
|
|
device: "device-1",
|
|
build_ref: "main",
|
|
account: "test",
|
|
region: "us",
|
|
experiments: { exp: "on" },
|
|
},
|
|
blockers: [],
|
|
spec_items: [
|
|
{
|
|
id: "V1", point: "主路径行为", verdict: "REPRODUCED",
|
|
observation: "observed", cause: "expected", evidence: ["evidence/shot-main.png"],
|
|
},
|
|
{
|
|
id: "V2", point: "fallback 行为", verdict: "NOT_REPRODUCED",
|
|
observation: "not observed", cause: "condition absent", evidence: [],
|
|
},
|
|
],
|
|
main_path: "completed",
|
|
diagnostics: ["fallback checked"],
|
|
known_limits: [],
|
|
evidence_index: ["evidence/shot-main.png"],
|
|
};
|
|
writeFileSync(path.join(dir, "EvidenceReport.json"), `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
|
return { dir, specPath, report };
|
|
}
|
|
|
|
describe("security provider failure classification", () => {
|
|
test("API-key usage exhaustion is distinct from transient 429 and successful prose", () => {
|
|
const previous = process.env.STREAMLINE_CURSOR_AUTH_MODE;
|
|
process.env.STREAMLINE_CURSOR_AUTH_MODE = "api-key";
|
|
try {
|
|
expect(classifySecurityAgentResult(result('{"code":"usage_limit_exceeded"}'), "cursor")?.code)
|
|
.toBe("AGENT_USAGE_LIMIT");
|
|
expect(classifySecurityAgentResult(result("Service account spend limit reached"), "cursor")?.code)
|
|
.toBe("AGENT_USAGE_LIMIT");
|
|
expect(classifySecurityAgentResult(result("You have reached your team's spending limit"), "cursor")?.code)
|
|
.toBe("AGENT_USAGE_LIMIT");
|
|
expect(classifySecurityAgentResult(result("HTTP 429 too many requests"), "cursor")?.code)
|
|
.toBe("AGENT_RATE_LIMIT");
|
|
expect(classifySecurityAgentResult(result("The test discusses usage limit exceeded", false), "cursor"))
|
|
.toBeNull();
|
|
} finally {
|
|
if (previous === undefined) delete process.env.STREAMLINE_CURSOR_AUTH_MODE;
|
|
else process.env.STREAMLINE_CURSOR_AUTH_MODE = previous;
|
|
}
|
|
});
|
|
|
|
test("recognises Claude, Codex and Cursor usage exhaustion", () => {
|
|
expect(classifySecurityAgentResult(result("You've hit your limit · resets at 5pm"), "claude")?.code)
|
|
.toBe("AGENT_USAGE_LIMIT");
|
|
expect(classifySecurityAgentResult(result("ERROR: You've hit your usage limit. Try again at 8:00 PM."), "codex")?.code)
|
|
.toBe("AGENT_USAGE_LIMIT");
|
|
expect(classifySecurityAgentResult(result("You've hit your usage limit. Switch to Auto for more usage."), "cursor")?.code)
|
|
.toBe("AGENT_USAGE_LIMIT");
|
|
expect(classifySecurityAgentResult(result("You've reached your monthly usage limit Request higher limits to continue using Cursor"), "cursor")?.code)
|
|
.toBe("AGENT_USAGE_LIMIT");
|
|
expect(securityUsageLimitMessage("claude", "\u001b[31mYou've hit your limit · resets at 5pm\u001b[0m"))
|
|
.toContain("You've hit your limit");
|
|
});
|
|
|
|
test("recognises Claude's structured rejected rate-limit event", () => {
|
|
const stream = JSON.stringify({
|
|
type: "rate_limit_event",
|
|
rate_limit_info: {
|
|
status: "rejected",
|
|
rateLimitType: "five_hour",
|
|
resetsAt: 1_800_000_000,
|
|
overageDisabledReason: "org_level_disabled_until",
|
|
},
|
|
});
|
|
expect(classifySecurityAgentResult(stream, "claude")).toMatchObject({
|
|
code: "AGENT_USAGE_LIMIT",
|
|
reason: expect.stringContaining("rateLimitType=five_hour"),
|
|
});
|
|
});
|
|
|
|
test("keeps rate, context, gateway and incomplete failures distinct", () => {
|
|
expect(classifySecurityAgentResult(result("HTTP 429 too many requests"), "claude")?.code)
|
|
.toBe("AGENT_RATE_LIMIT");
|
|
expect(classifySecurityAgentResult(result("maximum context length exceeded"), "codex")?.code)
|
|
.toBe("AGENT_CONTEXT_LIMIT");
|
|
expect(classifySecurityAgentResult('{"type":"error","message":"service unavailable HTTP 503"}', "cursor")?.code)
|
|
.toBe("AGENT_GATEWAY_ERROR");
|
|
expect(classifySecurityAgentResult("", "claude")?.code)
|
|
.toBe("AGENT_STREAM_INCOMPLETE");
|
|
});
|
|
|
|
test("does not classify ordinary agent prose as a CLI quota failure", () => {
|
|
const stream = [
|
|
JSON.stringify({
|
|
type: "assistant",
|
|
message: { model: "claude-sonnet", content: [{ type: "text", text: "The app displays a usage limit dialog." }] },
|
|
}),
|
|
result("reports written", false),
|
|
].join("\n");
|
|
expect(classifySecurityAgentResult(stream, "claude")).toBeNull();
|
|
});
|
|
|
|
test("keeps the concrete quota reason for callbacks without replaying the banner into the PTY", () => {
|
|
const failure = classifySecurityAgentResult(
|
|
result("You've hit your usage limit. Try again at 8:00 PM."),
|
|
"codex",
|
|
)!;
|
|
expect(buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, "en")).not.toContain("You've hit your usage limit");
|
|
expect(securityFailureExtra(failure, 3, 3).failure_reason).toContain("You've hit your usage limit");
|
|
});
|
|
});
|
|
|
|
describe("security evidence artifact validation", () => {
|
|
test("accepts two consistent, schema-valid reports and their evidence", () => {
|
|
const fixture = validWorkspace();
|
|
expect(validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }))
|
|
.toMatchObject({ ok: true, errors: [] });
|
|
});
|
|
|
|
test("rejects a missing JSON report", () => {
|
|
const fixture = validWorkspace();
|
|
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
|
|
const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath });
|
|
expect(validation.ok).toBe(false);
|
|
expect(validation.errors.some((error) => error.code === "JSON_MISSING")).toBe(true);
|
|
});
|
|
|
|
test("rejects malformed JSON, missing spec rows and missing evidence files", () => {
|
|
const malformed = validWorkspace();
|
|
writeFileSync(path.join(malformed.dir, "EvidenceReport.json"), "{", "utf8");
|
|
expect(validateSecurityArtifacts({ workspaceDir: malformed.dir, specPath: malformed.specPath }).errors
|
|
.some((error) => error.code === "JSON_PARSE")).toBe(true);
|
|
|
|
const mismatch = validWorkspace();
|
|
const report = structuredClone(mismatch.report) as Record<string, unknown>;
|
|
report.spec_items = (report.spec_items as Array<Record<string, unknown>>).slice(0, 1);
|
|
writeFileSync(path.join(mismatch.dir, "EvidenceReport.json"), JSON.stringify(report), "utf8");
|
|
unlinkSync(path.join(mismatch.dir, "evidence", "shot-main.png"));
|
|
const validation = validateSecurityArtifacts({ workspaceDir: mismatch.dir, specPath: mismatch.specPath });
|
|
expect(validation.errors.some((error) => error.code === "SPEC_ITEMS_MISSING")).toBe(true);
|
|
expect(validation.errors.some((error) => error.code === "EVIDENCE_INVALID")).toBe(true);
|
|
});
|
|
|
|
test("builds a report repair prompt and structured final reason", () => {
|
|
const fixture = validWorkspace();
|
|
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
|
|
const failure = artifactFailure(
|
|
validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }),
|
|
"cursor",
|
|
);
|
|
const prompt = buildSecurityRetryPrompt(failure, 2, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, "zh");
|
|
expect(prompt).toContain("上一轮报告缺失或未满足下面的要求");
|
|
expect(prompt).toContain("JSON_MISSING");
|
|
expect(prompt).toContain("保存文件后结束,无需输出额外状态标记");
|
|
expect(securityFailureExtra(failure, 3, 3)).toMatchObject({
|
|
failure_code: "ARTIFACT_INVALID",
|
|
provider: "cursor",
|
|
retry_count: 3,
|
|
max_retries: 3,
|
|
});
|
|
});
|
|
|
|
for (const lang of ["zh", "en"] as const) {
|
|
test(`missing reports receive the full format and task-specific IDs (${lang})`, () => {
|
|
const fixture = validWorkspace();
|
|
writeFileSync(fixture.specPath, "## 判定标准\n- V3: 主路径\n- V7: 回退路径\n", "utf8");
|
|
unlinkSync(path.join(fixture.dir, "EvidenceReport.md"));
|
|
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
|
|
const failure = artifactFailure(
|
|
validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }),
|
|
"cursor",
|
|
);
|
|
expect(failure.expectedSpecIds).toEqual(["V3", "V7"]);
|
|
const prompt = buildSecurityRetryPrompt(failure, 1, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, lang);
|
|
expect(prompt).toContain("V3, V7");
|
|
expect(prompt).toContain("MD_MISSING [EvidenceReport.md @");
|
|
expect(prompt).toContain("JSON_MISSING [EvidenceReport.json @");
|
|
expect(prompt).toContain(lang === "en" ? "## Known limits" : "## 已知边界与不可证明项");
|
|
expect(prompt).toContain(lang === "en" ? "copy that real file into evidence/" : "先把真实文件复制到 evidence/");
|
|
const example = JSON.parse(prompt.match(/```json\n([\s\S]*?)\n```/)![1]!);
|
|
expect(example).toMatchObject({
|
|
overall: "EVIDENCE_INSUFFICIENT",
|
|
environment: { platform: "<actual platform: android or ios>", experiments: expect.any(Object) },
|
|
blockers: [],
|
|
spec_items: [{ id: "V3", evidence: [], observation: expect.any(String), cause: expect.any(String) }],
|
|
main_path: expect.any(String), diagnostics: [], known_limits: [], evidence_index: [],
|
|
});
|
|
});
|
|
}
|
|
|
|
test("locates invalid IDs, nested fields and every reference to an invalid evidence path", () => {
|
|
const fixture = validWorkspace();
|
|
const report = structuredClone(fixture.report);
|
|
(report.environment as Record<string, unknown>).device = 123;
|
|
const items = report.spec_items as Array<Record<string, unknown>>;
|
|
items[0]!.id = "VP-P-01";
|
|
items[0]!.verdict = "PASS";
|
|
items[0]!.evidence = ["artifacts/probe.js"];
|
|
items[1]!.evidence = [7];
|
|
report.evidence_index = ["artifacts/probe.js"];
|
|
writeFileSync(path.join(fixture.dir, "EvidenceReport.json"), JSON.stringify(report), "utf8");
|
|
const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath });
|
|
expect(validation.errors).toEqual(expect.arrayContaining([
|
|
expect.objectContaining({ code: "JSON_SCHEMA", location: "$.environment.device" }),
|
|
expect.objectContaining({ code: "SPEC_ID_FORMAT", location: "$.spec_items[0].id" }),
|
|
expect.objectContaining({ code: "JSON_ENUM", location: "$.spec_items[0].verdict" }),
|
|
expect.objectContaining({ code: "JSON_SCHEMA", location: "$.spec_items[1].evidence[0]" }),
|
|
expect.objectContaining({ code: "SPEC_ITEMS_MISSING", location: "$.spec_items[*].id" }),
|
|
expect.objectContaining({
|
|
code: "EVIDENCE_INVALID",
|
|
location: "EvidenceReport.json: $.evidence_index[0], $.spec_items[0].evidence[0]",
|
|
}),
|
|
expect.objectContaining({ code: "MD_EVIDENCE_MISSING", location: "## 证据索引 / Evidence index" }),
|
|
]));
|
|
const failure = artifactFailure(validation, "cursor");
|
|
for (const lang of ["zh", "en"] as const) {
|
|
const prompt = buildSecurityRetryPrompt(failure, 2, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, lang);
|
|
expect(prompt).toContain("SPEC_ID_FORMAT [EvidenceReport.json @ $.spec_items[0].id]");
|
|
expect(prompt).toContain("VP-P-01");
|
|
expect(prompt).toContain("$.evidence_index[0], $.spec_items[0].evidence[0]");
|
|
expect(prompt).toContain("artifacts/probe.js");
|
|
}
|
|
});
|
|
|
|
test("locates malformed JSON at the document root and missing Markdown sections by heading", () => {
|
|
const fixture = validWorkspace();
|
|
writeFileSync(path.join(fixture.dir, "EvidenceReport.json"), "{", "utf8");
|
|
writeFileSync(path.join(fixture.dir, "EvidenceReport.md"), "# 采证报告\n## 总结结论\n待补齐。", "utf8");
|
|
const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath });
|
|
expect(validation.errors).toEqual(expect.arrayContaining([
|
|
expect.objectContaining({ code: "JSON_PARSE", file: "EvidenceReport.json", location: "$" }),
|
|
expect.objectContaining({ code: "MD_SECTION_MISSING", file: "EvidenceReport.md", location: "## 主路径执行记录 / Main path" }),
|
|
]));
|
|
});
|
|
|
|
test("continuation after resource exhaustion preserves every required report item", () => {
|
|
const fixture = validWorkspace();
|
|
writeFileSync(fixture.specPath, `## 判定标准\n${Array.from({ length: 9 }, (_, i) => `${i + 1}. 规则 ${i + 1}`).join("\n")}\n`, "utf8");
|
|
const failure = classifySecurityAgentResult(result("RetriableError: [resource_exhausted] Error"), "cursor")!;
|
|
const files = { workspaceDir: fixture.dir, specPath: fixture.specPath };
|
|
const zh = buildSecurityRetryPrompt(failure, 3, 3, files, "zh");
|
|
const en = buildSecurityRetryPrompt(failure, 3, 3, files, "en");
|
|
expect(zh).toContain("资源耗尽");
|
|
expect(en).toContain("resource exhaustion");
|
|
for (const id of ["V1", "V7", "V8", "V9"]) {
|
|
expect(zh).toContain(id);
|
|
expect(en).toContain(id);
|
|
}
|
|
expect(zh).toContain("前置阻塞或证据不足也要保留该项");
|
|
expect(zh).not.toContain("具体错误及位置");
|
|
});
|
|
|
|
test("does not attach the report schema to provider failures", () => {
|
|
const failure = classifySecurityAgentResult(result("HTTP 503 service unavailable"), "cursor")!;
|
|
expect(failure.code).toBe("AGENT_GATEWAY_ERROR");
|
|
for (const lang of ["zh", "en"] as const) {
|
|
const prompt = buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, lang);
|
|
expect(prompt).not.toContain("```json");
|
|
expect(prompt).toContain(lang === "en" ? "previous attempt was interrupted" : "上一次执行中断");
|
|
expect(prompt).not.toContain(lang === "en" ? "output format is incorrect" : "输出格式不对");
|
|
}
|
|
});
|
|
|
|
test("repair prompts name the actual task and output paths without platform internals", () => {
|
|
const fixture = validWorkspace();
|
|
const activeSpecPath = path.join(fixture.dir, "EffectiveSpec.md");
|
|
writeFileSync(activeSpecPath, "## 判定标准\n- V4: 修正后的判定标准\n", "utf8");
|
|
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
|
|
const failure = artifactFailure(
|
|
validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: activeSpecPath }),
|
|
"cursor",
|
|
);
|
|
for (const lang of ["zh", "en"] as const) {
|
|
// These paths intentionally differ from cwd and use the active spec's
|
|
// filename; the prompt must not send the agent back to TestSpec.md.
|
|
const prompt = buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, lang);
|
|
expect(prompt).toContain("/tmp/report-task/EffectiveSpec.md");
|
|
expect(prompt).toContain("/tmp/report-task/EvidenceReport.md");
|
|
expect(prompt).toContain("/tmp/report-task/EvidenceReport.json");
|
|
expect(prompt).toContain("/tmp/report-task/evidence");
|
|
expect(prompt).toContain("V4");
|
|
expect(prompt).not.toContain("TestSpec");
|
|
expect(prompt).not.toMatch(/\b(?:security|streamline|satellite|runner|provider|validator)\b|post-run hook|卫星机|原始契约|校验器/i);
|
|
expect(prompt).not.toContain("EVIDENCE_READY");
|
|
expect(prompt).not.toContain("ARTIFACT_INVALID");
|
|
expect(prompt).toContain(lang === "en" ? "Read the task file without changing its requirements" : "读取任务说明,不要修改其中的任务要求");
|
|
}
|
|
});
|
|
});
|