Files
streamline/satellite/test/security_run_policy.test.ts
T
2026-11-16 17:26:44 -08:00

357 lines
17 KiB
TypeScript

import { afterEach, describe, expect, test } from "bun:test";
import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { securityUsageLimitMessage } from "@bbbug/shared/security_agent_failure";
import {
artifactFailure,
buildSecurityRetryPrompt,
classifySecurityAgentResult,
securityFailureExtra,
validateSecurityArtifacts,
} from "../src/security_run_policy";
const dirs: string[] = [];
const retryTaskFiles = { workspaceDir: "/tmp/report-task", specPath: "/tmp/report-task/EffectiveSpec.md" };
afterEach(() => {
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
function result(message: string, isError = true): string {
return JSON.stringify({
type: "result",
subtype: isError ? "error_during_execution" : "success",
is_error: isError,
result: message,
});
}
function validWorkspace(): { dir: string; specPath: string; report: Record<string, unknown> } {
const dir = mkdtempSync(path.join(tmpdir(), "streamline-security-policy-"));
dirs.push(dir);
mkdirSync(path.join(dir, "evidence"), { recursive: true });
writeFileSync(path.join(dir, "evidence", "shot-main.png"), "png", "utf8");
const specPath = path.join(dir, "TestSpec.md");
writeFileSync(specPath, `# SecuritySpec
## 判定标准
1. 主路径行为
2. fallback 行为
## 其他
...
`, "utf8");
writeFileSync(path.join(dir, "EvidenceReport.md"), `# 采证报告
## 总结结论
已复现。
## SPEC 验证项逐项对比
| ID | 结论 | 证据 |
|---|---|---|
| V1 | 已复现 | evidence/shot-main.png |
| V2 | 未复现 | 无 |
## 主路径执行记录
已执行。
## 分支 / fallback / diagnostics
已检查。
## 已知边界与不可证明项
无。
## 证据索引
- evidence/shot-main.png
`, "utf8");
const report: Record<string, unknown> = {
overall: "PARTIALLY_REPRODUCED",
environment: {
platform: "android",
device: "device-1",
build_ref: "main",
account: "test",
region: "us",
experiments: { exp: "on" },
},
blockers: [],
spec_items: [
{
id: "V1", point: "主路径行为", verdict: "REPRODUCED",
observation: "observed", cause: "expected", evidence: ["evidence/shot-main.png"],
},
{
id: "V2", point: "fallback 行为", verdict: "NOT_REPRODUCED",
observation: "not observed", cause: "condition absent", evidence: [],
},
],
main_path: "completed",
diagnostics: ["fallback checked"],
known_limits: [],
evidence_index: ["evidence/shot-main.png"],
};
writeFileSync(path.join(dir, "EvidenceReport.json"), `${JSON.stringify(report, null, 2)}\n`, "utf8");
return { dir, specPath, report };
}
describe("security provider failure classification", () => {
test("API-key usage exhaustion is distinct from transient 429 and successful prose", () => {
const previous = process.env.STREAMLINE_CURSOR_AUTH_MODE;
process.env.STREAMLINE_CURSOR_AUTH_MODE = "api-key";
try {
expect(classifySecurityAgentResult(result('{"code":"usage_limit_exceeded"}'), "cursor")?.code)
.toBe("AGENT_USAGE_LIMIT");
expect(classifySecurityAgentResult(result("Service account spend limit reached"), "cursor")?.code)
.toBe("AGENT_USAGE_LIMIT");
expect(classifySecurityAgentResult(result("You have reached your team's spending limit"), "cursor")?.code)
.toBe("AGENT_USAGE_LIMIT");
expect(classifySecurityAgentResult(result("HTTP 429 too many requests"), "cursor")?.code)
.toBe("AGENT_RATE_LIMIT");
expect(classifySecurityAgentResult(result("The test discusses usage limit exceeded", false), "cursor"))
.toBeNull();
} finally {
if (previous === undefined) delete process.env.STREAMLINE_CURSOR_AUTH_MODE;
else process.env.STREAMLINE_CURSOR_AUTH_MODE = previous;
}
});
test("recognises Claude, Codex and Cursor usage exhaustion", () => {
expect(classifySecurityAgentResult(result("You've hit your limit · resets at 5pm"), "claude")?.code)
.toBe("AGENT_USAGE_LIMIT");
expect(classifySecurityAgentResult(result("ERROR: You've hit your usage limit. Try again at 8:00 PM."), "codex")?.code)
.toBe("AGENT_USAGE_LIMIT");
expect(classifySecurityAgentResult(result("You've hit your usage limit. Switch to Auto for more usage."), "cursor")?.code)
.toBe("AGENT_USAGE_LIMIT");
expect(classifySecurityAgentResult(result("You've reached your monthly usage limit Request higher limits to continue using Cursor"), "cursor")?.code)
.toBe("AGENT_USAGE_LIMIT");
expect(securityUsageLimitMessage("claude", "\u001b[31mYou've hit your limit · resets at 5pm\u001b[0m"))
.toContain("You've hit your limit");
});
test("recognises Claude's structured rejected rate-limit event", () => {
const stream = JSON.stringify({
type: "rate_limit_event",
rate_limit_info: {
status: "rejected",
rateLimitType: "five_hour",
resetsAt: 1_800_000_000,
overageDisabledReason: "org_level_disabled_until",
},
});
expect(classifySecurityAgentResult(stream, "claude")).toMatchObject({
code: "AGENT_USAGE_LIMIT",
reason: expect.stringContaining("rateLimitType=five_hour"),
});
});
test("keeps rate, context, gateway and incomplete failures distinct", () => {
expect(classifySecurityAgentResult(result("HTTP 429 too many requests"), "claude")?.code)
.toBe("AGENT_RATE_LIMIT");
expect(classifySecurityAgentResult(result("maximum context length exceeded"), "codex")?.code)
.toBe("AGENT_CONTEXT_LIMIT");
expect(classifySecurityAgentResult('{"type":"error","message":"service unavailable HTTP 503"}', "cursor")?.code)
.toBe("AGENT_GATEWAY_ERROR");
expect(classifySecurityAgentResult("", "claude")?.code)
.toBe("AGENT_STREAM_INCOMPLETE");
});
test("does not classify ordinary agent prose as a CLI quota failure", () => {
const stream = [
JSON.stringify({
type: "assistant",
message: { model: "claude-sonnet", content: [{ type: "text", text: "The app displays a usage limit dialog." }] },
}),
result("reports written", false),
].join("\n");
expect(classifySecurityAgentResult(stream, "claude")).toBeNull();
});
test("keeps the concrete quota reason for callbacks without replaying the banner into the PTY", () => {
const failure = classifySecurityAgentResult(
result("You've hit your usage limit. Try again at 8:00 PM."),
"codex",
)!;
expect(buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, "en")).not.toContain("You've hit your usage limit");
expect(securityFailureExtra(failure, 3, 3).failure_reason).toContain("You've hit your usage limit");
});
});
describe("security evidence artifact validation", () => {
test("accepts two consistent, schema-valid reports and their evidence", () => {
const fixture = validWorkspace();
expect(validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }))
.toMatchObject({ ok: true, errors: [] });
});
test("rejects a missing JSON report", () => {
const fixture = validWorkspace();
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath });
expect(validation.ok).toBe(false);
expect(validation.errors.some((error) => error.code === "JSON_MISSING")).toBe(true);
});
test("rejects malformed JSON, missing spec rows and missing evidence files", () => {
const malformed = validWorkspace();
writeFileSync(path.join(malformed.dir, "EvidenceReport.json"), "{", "utf8");
expect(validateSecurityArtifacts({ workspaceDir: malformed.dir, specPath: malformed.specPath }).errors
.some((error) => error.code === "JSON_PARSE")).toBe(true);
const mismatch = validWorkspace();
const report = structuredClone(mismatch.report) as Record<string, unknown>;
report.spec_items = (report.spec_items as Array<Record<string, unknown>>).slice(0, 1);
writeFileSync(path.join(mismatch.dir, "EvidenceReport.json"), JSON.stringify(report), "utf8");
unlinkSync(path.join(mismatch.dir, "evidence", "shot-main.png"));
const validation = validateSecurityArtifacts({ workspaceDir: mismatch.dir, specPath: mismatch.specPath });
expect(validation.errors.some((error) => error.code === "SPEC_ITEMS_MISSING")).toBe(true);
expect(validation.errors.some((error) => error.code === "EVIDENCE_INVALID")).toBe(true);
});
test("builds a report repair prompt and structured final reason", () => {
const fixture = validWorkspace();
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
const failure = artifactFailure(
validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }),
"cursor",
);
const prompt = buildSecurityRetryPrompt(failure, 2, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, "zh");
expect(prompt).toContain("上一轮报告缺失或未满足下面的要求");
expect(prompt).toContain("JSON_MISSING");
expect(prompt).toContain("保存文件后结束,无需输出额外状态标记");
expect(securityFailureExtra(failure, 3, 3)).toMatchObject({
failure_code: "ARTIFACT_INVALID",
provider: "cursor",
retry_count: 3,
max_retries: 3,
});
});
for (const lang of ["zh", "en"] as const) {
test(`missing reports receive the full format and task-specific IDs (${lang})`, () => {
const fixture = validWorkspace();
writeFileSync(fixture.specPath, "## 判定标准\n- V3: 主路径\n- V7: 回退路径\n", "utf8");
unlinkSync(path.join(fixture.dir, "EvidenceReport.md"));
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
const failure = artifactFailure(
validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }),
"cursor",
);
expect(failure.expectedSpecIds).toEqual(["V3", "V7"]);
const prompt = buildSecurityRetryPrompt(failure, 1, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, lang);
expect(prompt).toContain("V3, V7");
expect(prompt).toContain("MD_MISSING [EvidenceReport.md @");
expect(prompt).toContain("JSON_MISSING [EvidenceReport.json @");
expect(prompt).toContain(lang === "en" ? "## Known limits" : "## 已知边界与不可证明项");
expect(prompt).toContain(lang === "en" ? "copy that real file into evidence/" : "先把真实文件复制到 evidence/");
const example = JSON.parse(prompt.match(/```json\n([\s\S]*?)\n```/)![1]!);
expect(example).toMatchObject({
overall: "EVIDENCE_INSUFFICIENT",
environment: { platform: "<actual platform: android or ios>", experiments: expect.any(Object) },
blockers: [],
spec_items: [{ id: "V3", evidence: [], observation: expect.any(String), cause: expect.any(String) }],
main_path: expect.any(String), diagnostics: [], known_limits: [], evidence_index: [],
});
});
}
test("locates invalid IDs, nested fields and every reference to an invalid evidence path", () => {
const fixture = validWorkspace();
const report = structuredClone(fixture.report);
(report.environment as Record<string, unknown>).device = 123;
const items = report.spec_items as Array<Record<string, unknown>>;
items[0]!.id = "VP-P-01";
items[0]!.verdict = "PASS";
items[0]!.evidence = ["artifacts/probe.js"];
items[1]!.evidence = [7];
report.evidence_index = ["artifacts/probe.js"];
writeFileSync(path.join(fixture.dir, "EvidenceReport.json"), JSON.stringify(report), "utf8");
const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath });
expect(validation.errors).toEqual(expect.arrayContaining([
expect.objectContaining({ code: "JSON_SCHEMA", location: "$.environment.device" }),
expect.objectContaining({ code: "SPEC_ID_FORMAT", location: "$.spec_items[0].id" }),
expect.objectContaining({ code: "JSON_ENUM", location: "$.spec_items[0].verdict" }),
expect.objectContaining({ code: "JSON_SCHEMA", location: "$.spec_items[1].evidence[0]" }),
expect.objectContaining({ code: "SPEC_ITEMS_MISSING", location: "$.spec_items[*].id" }),
expect.objectContaining({
code: "EVIDENCE_INVALID",
location: "EvidenceReport.json: $.evidence_index[0], $.spec_items[0].evidence[0]",
}),
expect.objectContaining({ code: "MD_EVIDENCE_MISSING", location: "## 证据索引 / Evidence index" }),
]));
const failure = artifactFailure(validation, "cursor");
for (const lang of ["zh", "en"] as const) {
const prompt = buildSecurityRetryPrompt(failure, 2, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, lang);
expect(prompt).toContain("SPEC_ID_FORMAT [EvidenceReport.json @ $.spec_items[0].id]");
expect(prompt).toContain("VP-P-01");
expect(prompt).toContain("$.evidence_index[0], $.spec_items[0].evidence[0]");
expect(prompt).toContain("artifacts/probe.js");
}
});
test("locates malformed JSON at the document root and missing Markdown sections by heading", () => {
const fixture = validWorkspace();
writeFileSync(path.join(fixture.dir, "EvidenceReport.json"), "{", "utf8");
writeFileSync(path.join(fixture.dir, "EvidenceReport.md"), "# 采证报告\n## 总结结论\n待补齐。", "utf8");
const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath });
expect(validation.errors).toEqual(expect.arrayContaining([
expect.objectContaining({ code: "JSON_PARSE", file: "EvidenceReport.json", location: "$" }),
expect.objectContaining({ code: "MD_SECTION_MISSING", file: "EvidenceReport.md", location: "## 主路径执行记录 / Main path" }),
]));
});
test("continuation after resource exhaustion preserves every required report item", () => {
const fixture = validWorkspace();
writeFileSync(fixture.specPath, `## 判定标准\n${Array.from({ length: 9 }, (_, i) => `${i + 1}. 规则 ${i + 1}`).join("\n")}\n`, "utf8");
const failure = classifySecurityAgentResult(result("RetriableError: [resource_exhausted] Error"), "cursor")!;
const files = { workspaceDir: fixture.dir, specPath: fixture.specPath };
const zh = buildSecurityRetryPrompt(failure, 3, 3, files, "zh");
const en = buildSecurityRetryPrompt(failure, 3, 3, files, "en");
expect(zh).toContain("资源耗尽");
expect(en).toContain("resource exhaustion");
for (const id of ["V1", "V7", "V8", "V9"]) {
expect(zh).toContain(id);
expect(en).toContain(id);
}
expect(zh).toContain("前置阻塞或证据不足也要保留该项");
expect(zh).not.toContain("具体错误及位置");
});
test("does not attach the report schema to provider failures", () => {
const failure = classifySecurityAgentResult(result("HTTP 503 service unavailable"), "cursor")!;
expect(failure.code).toBe("AGENT_GATEWAY_ERROR");
for (const lang of ["zh", "en"] as const) {
const prompt = buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, lang);
expect(prompt).not.toContain("```json");
expect(prompt).toContain(lang === "en" ? "previous attempt was interrupted" : "上一次执行中断");
expect(prompt).not.toContain(lang === "en" ? "output format is incorrect" : "输出格式不对");
}
});
test("repair prompts name the actual task and output paths without platform internals", () => {
const fixture = validWorkspace();
const activeSpecPath = path.join(fixture.dir, "EffectiveSpec.md");
writeFileSync(activeSpecPath, "## 判定标准\n- V4: 修正后的判定标准\n", "utf8");
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
const failure = artifactFailure(
validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: activeSpecPath }),
"cursor",
);
for (const lang of ["zh", "en"] as const) {
// These paths intentionally differ from cwd and use the active spec's
// filename; the prompt must not send the agent back to TestSpec.md.
const prompt = buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, lang);
expect(prompt).toContain("/tmp/report-task/EffectiveSpec.md");
expect(prompt).toContain("/tmp/report-task/EvidenceReport.md");
expect(prompt).toContain("/tmp/report-task/EvidenceReport.json");
expect(prompt).toContain("/tmp/report-task/evidence");
expect(prompt).toContain("V4");
expect(prompt).not.toContain("TestSpec");
expect(prompt).not.toMatch(/\b(?:security|streamline|satellite|runner|provider|validator)\b|post-run hook|卫星机|原始契约|校验器/i);
expect(prompt).not.toContain("EVIDENCE_READY");
expect(prompt).not.toContain("ARTIFACT_INVALID");
expect(prompt).toContain(lang === "en" ? "Read the task file without changing its requirements" : "读取任务说明,不要修改其中的任务要求");
}
});
});