Initial import of streamline
This commit is contained in:
@@ -0,0 +1,356 @@
|
||||
import { afterEach, describe, expect, test } from "bun:test";
|
||||
import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { securityUsageLimitMessage } from "@bbbug/shared/security_agent_failure";
|
||||
import {
|
||||
artifactFailure,
|
||||
buildSecurityRetryPrompt,
|
||||
classifySecurityAgentResult,
|
||||
securityFailureExtra,
|
||||
validateSecurityArtifacts,
|
||||
} from "../src/security_run_policy";
|
||||
|
||||
const dirs: string[] = [];
|
||||
const retryTaskFiles = { workspaceDir: "/tmp/report-task", specPath: "/tmp/report-task/EffectiveSpec.md" };
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function result(message: string, isError = true): string {
|
||||
return JSON.stringify({
|
||||
type: "result",
|
||||
subtype: isError ? "error_during_execution" : "success",
|
||||
is_error: isError,
|
||||
result: message,
|
||||
});
|
||||
}
|
||||
|
||||
function validWorkspace(): { dir: string; specPath: string; report: Record<string, unknown> } {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "streamline-security-policy-"));
|
||||
dirs.push(dir);
|
||||
mkdirSync(path.join(dir, "evidence"), { recursive: true });
|
||||
writeFileSync(path.join(dir, "evidence", "shot-main.png"), "png", "utf8");
|
||||
const specPath = path.join(dir, "TestSpec.md");
|
||||
writeFileSync(specPath, `# SecuritySpec
|
||||
|
||||
## 判定标准
|
||||
1. 主路径行为
|
||||
2. fallback 行为
|
||||
|
||||
## 其他
|
||||
...
|
||||
`, "utf8");
|
||||
writeFileSync(path.join(dir, "EvidenceReport.md"), `# 采证报告
|
||||
|
||||
## 总结结论
|
||||
已复现。
|
||||
|
||||
## SPEC 验证项逐项对比
|
||||
| ID | 结论 | 证据 |
|
||||
|---|---|---|
|
||||
| V1 | 已复现 | evidence/shot-main.png |
|
||||
| V2 | 未复现 | 无 |
|
||||
|
||||
## 主路径执行记录
|
||||
已执行。
|
||||
|
||||
## 分支 / fallback / diagnostics
|
||||
已检查。
|
||||
|
||||
## 已知边界与不可证明项
|
||||
无。
|
||||
|
||||
## 证据索引
|
||||
- evidence/shot-main.png
|
||||
`, "utf8");
|
||||
const report: Record<string, unknown> = {
|
||||
overall: "PARTIALLY_REPRODUCED",
|
||||
environment: {
|
||||
platform: "android",
|
||||
device: "device-1",
|
||||
build_ref: "main",
|
||||
account: "test",
|
||||
region: "us",
|
||||
experiments: { exp: "on" },
|
||||
},
|
||||
blockers: [],
|
||||
spec_items: [
|
||||
{
|
||||
id: "V1", point: "主路径行为", verdict: "REPRODUCED",
|
||||
observation: "observed", cause: "expected", evidence: ["evidence/shot-main.png"],
|
||||
},
|
||||
{
|
||||
id: "V2", point: "fallback 行为", verdict: "NOT_REPRODUCED",
|
||||
observation: "not observed", cause: "condition absent", evidence: [],
|
||||
},
|
||||
],
|
||||
main_path: "completed",
|
||||
diagnostics: ["fallback checked"],
|
||||
known_limits: [],
|
||||
evidence_index: ["evidence/shot-main.png"],
|
||||
};
|
||||
writeFileSync(path.join(dir, "EvidenceReport.json"), `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
return { dir, specPath, report };
|
||||
}
|
||||
|
||||
describe("security provider failure classification", () => {
|
||||
test("API-key usage exhaustion is distinct from transient 429 and successful prose", () => {
|
||||
const previous = process.env.STREAMLINE_CURSOR_AUTH_MODE;
|
||||
process.env.STREAMLINE_CURSOR_AUTH_MODE = "api-key";
|
||||
try {
|
||||
expect(classifySecurityAgentResult(result('{"code":"usage_limit_exceeded"}'), "cursor")?.code)
|
||||
.toBe("AGENT_USAGE_LIMIT");
|
||||
expect(classifySecurityAgentResult(result("Service account spend limit reached"), "cursor")?.code)
|
||||
.toBe("AGENT_USAGE_LIMIT");
|
||||
expect(classifySecurityAgentResult(result("You have reached your team's spending limit"), "cursor")?.code)
|
||||
.toBe("AGENT_USAGE_LIMIT");
|
||||
expect(classifySecurityAgentResult(result("HTTP 429 too many requests"), "cursor")?.code)
|
||||
.toBe("AGENT_RATE_LIMIT");
|
||||
expect(classifySecurityAgentResult(result("The test discusses usage limit exceeded", false), "cursor"))
|
||||
.toBeNull();
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.STREAMLINE_CURSOR_AUTH_MODE;
|
||||
else process.env.STREAMLINE_CURSOR_AUTH_MODE = previous;
|
||||
}
|
||||
});
|
||||
|
||||
test("recognises Claude, Codex and Cursor usage exhaustion", () => {
|
||||
expect(classifySecurityAgentResult(result("You've hit your limit · resets at 5pm"), "claude")?.code)
|
||||
.toBe("AGENT_USAGE_LIMIT");
|
||||
expect(classifySecurityAgentResult(result("ERROR: You've hit your usage limit. Try again at 8:00 PM."), "codex")?.code)
|
||||
.toBe("AGENT_USAGE_LIMIT");
|
||||
expect(classifySecurityAgentResult(result("You've hit your usage limit. Switch to Auto for more usage."), "cursor")?.code)
|
||||
.toBe("AGENT_USAGE_LIMIT");
|
||||
expect(classifySecurityAgentResult(result("You've reached your monthly usage limit Request higher limits to continue using Cursor"), "cursor")?.code)
|
||||
.toBe("AGENT_USAGE_LIMIT");
|
||||
expect(securityUsageLimitMessage("claude", "\u001b[31mYou've hit your limit · resets at 5pm\u001b[0m"))
|
||||
.toContain("You've hit your limit");
|
||||
});
|
||||
|
||||
test("recognises Claude's structured rejected rate-limit event", () => {
|
||||
const stream = JSON.stringify({
|
||||
type: "rate_limit_event",
|
||||
rate_limit_info: {
|
||||
status: "rejected",
|
||||
rateLimitType: "five_hour",
|
||||
resetsAt: 1_800_000_000,
|
||||
overageDisabledReason: "org_level_disabled_until",
|
||||
},
|
||||
});
|
||||
expect(classifySecurityAgentResult(stream, "claude")).toMatchObject({
|
||||
code: "AGENT_USAGE_LIMIT",
|
||||
reason: expect.stringContaining("rateLimitType=five_hour"),
|
||||
});
|
||||
});
|
||||
|
||||
test("keeps rate, context, gateway and incomplete failures distinct", () => {
|
||||
expect(classifySecurityAgentResult(result("HTTP 429 too many requests"), "claude")?.code)
|
||||
.toBe("AGENT_RATE_LIMIT");
|
||||
expect(classifySecurityAgentResult(result("maximum context length exceeded"), "codex")?.code)
|
||||
.toBe("AGENT_CONTEXT_LIMIT");
|
||||
expect(classifySecurityAgentResult('{"type":"error","message":"service unavailable HTTP 503"}', "cursor")?.code)
|
||||
.toBe("AGENT_GATEWAY_ERROR");
|
||||
expect(classifySecurityAgentResult("", "claude")?.code)
|
||||
.toBe("AGENT_STREAM_INCOMPLETE");
|
||||
});
|
||||
|
||||
test("does not classify ordinary agent prose as a CLI quota failure", () => {
|
||||
const stream = [
|
||||
JSON.stringify({
|
||||
type: "assistant",
|
||||
message: { model: "claude-sonnet", content: [{ type: "text", text: "The app displays a usage limit dialog." }] },
|
||||
}),
|
||||
result("reports written", false),
|
||||
].join("\n");
|
||||
expect(classifySecurityAgentResult(stream, "claude")).toBeNull();
|
||||
});
|
||||
|
||||
test("keeps the concrete quota reason for callbacks without replaying the banner into the PTY", () => {
|
||||
const failure = classifySecurityAgentResult(
|
||||
result("You've hit your usage limit. Try again at 8:00 PM."),
|
||||
"codex",
|
||||
)!;
|
||||
expect(buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, "en")).not.toContain("You've hit your usage limit");
|
||||
expect(securityFailureExtra(failure, 3, 3).failure_reason).toContain("You've hit your usage limit");
|
||||
});
|
||||
});
|
||||
|
||||
describe("security evidence artifact validation", () => {
|
||||
test("accepts two consistent, schema-valid reports and their evidence", () => {
|
||||
const fixture = validWorkspace();
|
||||
expect(validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }))
|
||||
.toMatchObject({ ok: true, errors: [] });
|
||||
});
|
||||
|
||||
test("rejects a missing JSON report", () => {
|
||||
const fixture = validWorkspace();
|
||||
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
|
||||
const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath });
|
||||
expect(validation.ok).toBe(false);
|
||||
expect(validation.errors.some((error) => error.code === "JSON_MISSING")).toBe(true);
|
||||
});
|
||||
|
||||
test("rejects malformed JSON, missing spec rows and missing evidence files", () => {
|
||||
const malformed = validWorkspace();
|
||||
writeFileSync(path.join(malformed.dir, "EvidenceReport.json"), "{", "utf8");
|
||||
expect(validateSecurityArtifacts({ workspaceDir: malformed.dir, specPath: malformed.specPath }).errors
|
||||
.some((error) => error.code === "JSON_PARSE")).toBe(true);
|
||||
|
||||
const mismatch = validWorkspace();
|
||||
const report = structuredClone(mismatch.report) as Record<string, unknown>;
|
||||
report.spec_items = (report.spec_items as Array<Record<string, unknown>>).slice(0, 1);
|
||||
writeFileSync(path.join(mismatch.dir, "EvidenceReport.json"), JSON.stringify(report), "utf8");
|
||||
unlinkSync(path.join(mismatch.dir, "evidence", "shot-main.png"));
|
||||
const validation = validateSecurityArtifacts({ workspaceDir: mismatch.dir, specPath: mismatch.specPath });
|
||||
expect(validation.errors.some((error) => error.code === "SPEC_ITEMS_MISSING")).toBe(true);
|
||||
expect(validation.errors.some((error) => error.code === "EVIDENCE_INVALID")).toBe(true);
|
||||
});
|
||||
|
||||
test("builds a report repair prompt and structured final reason", () => {
|
||||
const fixture = validWorkspace();
|
||||
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
|
||||
const failure = artifactFailure(
|
||||
validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }),
|
||||
"cursor",
|
||||
);
|
||||
const prompt = buildSecurityRetryPrompt(failure, 2, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, "zh");
|
||||
expect(prompt).toContain("上一轮报告缺失或未满足下面的要求");
|
||||
expect(prompt).toContain("JSON_MISSING");
|
||||
expect(prompt).toContain("保存文件后结束,无需输出额外状态标记");
|
||||
expect(securityFailureExtra(failure, 3, 3)).toMatchObject({
|
||||
failure_code: "ARTIFACT_INVALID",
|
||||
provider: "cursor",
|
||||
retry_count: 3,
|
||||
max_retries: 3,
|
||||
});
|
||||
});
|
||||
|
||||
for (const lang of ["zh", "en"] as const) {
|
||||
test(`missing reports receive the full format and task-specific IDs (${lang})`, () => {
|
||||
const fixture = validWorkspace();
|
||||
writeFileSync(fixture.specPath, "## 判定标准\n- V3: 主路径\n- V7: 回退路径\n", "utf8");
|
||||
unlinkSync(path.join(fixture.dir, "EvidenceReport.md"));
|
||||
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
|
||||
const failure = artifactFailure(
|
||||
validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath }),
|
||||
"cursor",
|
||||
);
|
||||
expect(failure.expectedSpecIds).toEqual(["V3", "V7"]);
|
||||
const prompt = buildSecurityRetryPrompt(failure, 1, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, lang);
|
||||
expect(prompt).toContain("V3, V7");
|
||||
expect(prompt).toContain("MD_MISSING [EvidenceReport.md @");
|
||||
expect(prompt).toContain("JSON_MISSING [EvidenceReport.json @");
|
||||
expect(prompt).toContain(lang === "en" ? "## Known limits" : "## 已知边界与不可证明项");
|
||||
expect(prompt).toContain(lang === "en" ? "copy that real file into evidence/" : "先把真实文件复制到 evidence/");
|
||||
const example = JSON.parse(prompt.match(/```json\n([\s\S]*?)\n```/)![1]!);
|
||||
expect(example).toMatchObject({
|
||||
overall: "EVIDENCE_INSUFFICIENT",
|
||||
environment: { platform: "<actual platform: android or ios>", experiments: expect.any(Object) },
|
||||
blockers: [],
|
||||
spec_items: [{ id: "V3", evidence: [], observation: expect.any(String), cause: expect.any(String) }],
|
||||
main_path: expect.any(String), diagnostics: [], known_limits: [], evidence_index: [],
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test("locates invalid IDs, nested fields and every reference to an invalid evidence path", () => {
|
||||
const fixture = validWorkspace();
|
||||
const report = structuredClone(fixture.report);
|
||||
(report.environment as Record<string, unknown>).device = 123;
|
||||
const items = report.spec_items as Array<Record<string, unknown>>;
|
||||
items[0]!.id = "VP-P-01";
|
||||
items[0]!.verdict = "PASS";
|
||||
items[0]!.evidence = ["artifacts/probe.js"];
|
||||
items[1]!.evidence = [7];
|
||||
report.evidence_index = ["artifacts/probe.js"];
|
||||
writeFileSync(path.join(fixture.dir, "EvidenceReport.json"), JSON.stringify(report), "utf8");
|
||||
const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath });
|
||||
expect(validation.errors).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ code: "JSON_SCHEMA", location: "$.environment.device" }),
|
||||
expect.objectContaining({ code: "SPEC_ID_FORMAT", location: "$.spec_items[0].id" }),
|
||||
expect.objectContaining({ code: "JSON_ENUM", location: "$.spec_items[0].verdict" }),
|
||||
expect.objectContaining({ code: "JSON_SCHEMA", location: "$.spec_items[1].evidence[0]" }),
|
||||
expect.objectContaining({ code: "SPEC_ITEMS_MISSING", location: "$.spec_items[*].id" }),
|
||||
expect.objectContaining({
|
||||
code: "EVIDENCE_INVALID",
|
||||
location: "EvidenceReport.json: $.evidence_index[0], $.spec_items[0].evidence[0]",
|
||||
}),
|
||||
expect.objectContaining({ code: "MD_EVIDENCE_MISSING", location: "## 证据索引 / Evidence index" }),
|
||||
]));
|
||||
const failure = artifactFailure(validation, "cursor");
|
||||
for (const lang of ["zh", "en"] as const) {
|
||||
const prompt = buildSecurityRetryPrompt(failure, 2, 3, { workspaceDir: fixture.dir, specPath: fixture.specPath }, lang);
|
||||
expect(prompt).toContain("SPEC_ID_FORMAT [EvidenceReport.json @ $.spec_items[0].id]");
|
||||
expect(prompt).toContain("VP-P-01");
|
||||
expect(prompt).toContain("$.evidence_index[0], $.spec_items[0].evidence[0]");
|
||||
expect(prompt).toContain("artifacts/probe.js");
|
||||
}
|
||||
});
|
||||
|
||||
test("locates malformed JSON at the document root and missing Markdown sections by heading", () => {
|
||||
const fixture = validWorkspace();
|
||||
writeFileSync(path.join(fixture.dir, "EvidenceReport.json"), "{", "utf8");
|
||||
writeFileSync(path.join(fixture.dir, "EvidenceReport.md"), "# 采证报告\n## 总结结论\n待补齐。", "utf8");
|
||||
const validation = validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: fixture.specPath });
|
||||
expect(validation.errors).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ code: "JSON_PARSE", file: "EvidenceReport.json", location: "$" }),
|
||||
expect.objectContaining({ code: "MD_SECTION_MISSING", file: "EvidenceReport.md", location: "## 主路径执行记录 / Main path" }),
|
||||
]));
|
||||
});
|
||||
|
||||
test("continuation after resource exhaustion preserves every required report item", () => {
|
||||
const fixture = validWorkspace();
|
||||
writeFileSync(fixture.specPath, `## 判定标准\n${Array.from({ length: 9 }, (_, i) => `${i + 1}. 规则 ${i + 1}`).join("\n")}\n`, "utf8");
|
||||
const failure = classifySecurityAgentResult(result("RetriableError: [resource_exhausted] Error"), "cursor")!;
|
||||
const files = { workspaceDir: fixture.dir, specPath: fixture.specPath };
|
||||
const zh = buildSecurityRetryPrompt(failure, 3, 3, files, "zh");
|
||||
const en = buildSecurityRetryPrompt(failure, 3, 3, files, "en");
|
||||
expect(zh).toContain("资源耗尽");
|
||||
expect(en).toContain("resource exhaustion");
|
||||
for (const id of ["V1", "V7", "V8", "V9"]) {
|
||||
expect(zh).toContain(id);
|
||||
expect(en).toContain(id);
|
||||
}
|
||||
expect(zh).toContain("前置阻塞或证据不足也要保留该项");
|
||||
expect(zh).not.toContain("具体错误及位置");
|
||||
});
|
||||
|
||||
test("does not attach the report schema to provider failures", () => {
|
||||
const failure = classifySecurityAgentResult(result("HTTP 503 service unavailable"), "cursor")!;
|
||||
expect(failure.code).toBe("AGENT_GATEWAY_ERROR");
|
||||
for (const lang of ["zh", "en"] as const) {
|
||||
const prompt = buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, lang);
|
||||
expect(prompt).not.toContain("```json");
|
||||
expect(prompt).toContain(lang === "en" ? "previous attempt was interrupted" : "上一次执行中断");
|
||||
expect(prompt).not.toContain(lang === "en" ? "output format is incorrect" : "输出格式不对");
|
||||
}
|
||||
});
|
||||
|
||||
test("repair prompts name the actual task and output paths without platform internals", () => {
|
||||
const fixture = validWorkspace();
|
||||
const activeSpecPath = path.join(fixture.dir, "EffectiveSpec.md");
|
||||
writeFileSync(activeSpecPath, "## 判定标准\n- V4: 修正后的判定标准\n", "utf8");
|
||||
unlinkSync(path.join(fixture.dir, "EvidenceReport.json"));
|
||||
const failure = artifactFailure(
|
||||
validateSecurityArtifacts({ workspaceDir: fixture.dir, specPath: activeSpecPath }),
|
||||
"cursor",
|
||||
);
|
||||
for (const lang of ["zh", "en"] as const) {
|
||||
// These paths intentionally differ from cwd and use the active spec's
|
||||
// filename; the prompt must not send the agent back to TestSpec.md.
|
||||
const prompt = buildSecurityRetryPrompt(failure, 1, 3, retryTaskFiles, lang);
|
||||
expect(prompt).toContain("/tmp/report-task/EffectiveSpec.md");
|
||||
expect(prompt).toContain("/tmp/report-task/EvidenceReport.md");
|
||||
expect(prompt).toContain("/tmp/report-task/EvidenceReport.json");
|
||||
expect(prompt).toContain("/tmp/report-task/evidence");
|
||||
expect(prompt).toContain("V4");
|
||||
expect(prompt).not.toContain("TestSpec");
|
||||
expect(prompt).not.toMatch(/\b(?:security|streamline|satellite|runner|provider|validator)\b|post-run hook|卫星机|原始契约|校验器/i);
|
||||
expect(prompt).not.toContain("EVIDENCE_READY");
|
||||
expect(prompt).not.toContain("ARTIFACT_INVALID");
|
||||
expect(prompt).toContain(lang === "en" ? "Read the task file without changing its requirements" : "读取任务说明,不要修改其中的任务要求");
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user