140 lines
8.0 KiB
TypeScript
140 lines
8.0 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
|
import { readFileSync, mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { SecurityFailureStreak, type SecurityExecutionOutcome } from "../src/security_telemetry/failure_streak";
|
|
|
|
const root = path.resolve(import.meta.dir, "../..");
|
|
const modulePath = path.join(root, "satellite/src/security_telemetry/failure_streak.ts");
|
|
|
|
function outcome(taskId: string, phase: "evaluate" | "run" = "run", result: "failed" | "succeeded" = "failed", startedAt = 1): SecurityExecutionOutcome {
|
|
return { taskId, phase, outcome: result, startedAt };
|
|
}
|
|
|
|
describe("Security failed-task exit guard", () => {
|
|
test("evaluate and run share a three-distinct-task threshold", () => {
|
|
const streak = new SecurityFailureStreak();
|
|
expect(streak.record(outcome("a", "evaluate"))).toBe(false);
|
|
expect(streak.record(outcome("b", "run"))).toBe(false);
|
|
expect(streak.record(outcome("c", "evaluate"))).toBe(true);
|
|
expect(streak.snapshot()).toMatchObject({ exitRequested: true, failedTasks: [
|
|
{ taskId: "a", phase: "evaluate" }, { taskId: "b", phase: "run" }, { taskId: "c", phase: "evaluate" },
|
|
] });
|
|
expect(streak.record(outcome("d"))).toBe(false);
|
|
});
|
|
|
|
test("repeated failures and both stages of one task still count once", () => {
|
|
const streak = new SecurityFailureStreak();
|
|
streak.record(outcome("a", "evaluate"));
|
|
streak.record(outcome("a", "evaluate", "failed", 2));
|
|
streak.record(outcome("a", "run"));
|
|
expect(streak.snapshot().failedTasks.length).toBe(1);
|
|
expect(streak.record(outcome("b"))).toBe(false);
|
|
expect(streak.record(outcome("c"))).toBe(true);
|
|
});
|
|
|
|
test("a successful stage clears the streak and duplicate success cannot clear later failures", () => {
|
|
const streak = new SecurityFailureStreak();
|
|
streak.record(outcome("a"));
|
|
streak.record(outcome("b"));
|
|
streak.record(outcome("ready", "evaluate", "succeeded"));
|
|
expect(streak.snapshot().failedTasks).toEqual([]);
|
|
streak.record(outcome("c"));
|
|
streak.record(outcome("ready", "evaluate", "succeeded"));
|
|
expect(streak.snapshot().failedTasks.length).toBe(1);
|
|
expect(streak.record(outcome("d"))).toBe(false);
|
|
expect(streak.record(outcome("e"))).toBe(true);
|
|
});
|
|
|
|
test("success after the third failure cannot rescind the requested exit", () => {
|
|
const streak = new SecurityFailureStreak();
|
|
for (const id of ["a", "b", "c"]) streak.record(outcome(id));
|
|
streak.record(outcome("ready", "run", "succeeded"));
|
|
expect(streak.snapshot().exitRequested).toBe(true);
|
|
expect(new SecurityFailureStreak().snapshot().failedTasks).toEqual([]);
|
|
});
|
|
|
|
test("inactive control-plane imports never request process exit", async () => {
|
|
const script = `import { noteSecurityExecutionOutcome } from ${JSON.stringify(modulePath)};
|
|
for (const taskId of ["a", "b", "c"]) noteSecurityExecutionOutcome({ taskId, phase:"run", startedAt:1, outcome:"failed" });
|
|
console.log("control-plane-survived");`;
|
|
const proc = Bun.spawn([process.execPath, "-e", script], { stdout: "pipe", stderr: "pipe" });
|
|
const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
|
|
expect(code).toBe(0);
|
|
expect(stderr).toBe("");
|
|
expect(stdout).toContain("control-plane-survived");
|
|
});
|
|
|
|
test("third failure callback is durable before exit and status writes can finish", async () => {
|
|
const stateDir = mkdtempSync(path.join(tmpdir(), "security-failure-exit-"));
|
|
try {
|
|
const script = `import { initSecurityFailureExitGuard, trackSecurityStatusWrite, flushSecurityStatusWrites } from ${JSON.stringify(modulePath)};
|
|
import { fireSecurityCallback } from ${JSON.stringify(path.join(root, "satellite/src/security_telemetry/callback.ts"))};
|
|
initSecurityFailureExitGuard(async () => {
|
|
await flushSecurityStatusWrites(); console.log("status-flushed"); process.exit(1);
|
|
});
|
|
trackSecurityStatusWrite(new Promise(resolve => setTimeout(resolve, 30)));
|
|
for (const id of ["a", "b", "c"]) fireSecurityCallback({
|
|
event:"security_failed", security:{id,title:id,platform:"android",attempts:0},status:"security_failed",
|
|
phase:id==="b"?"evaluate":"run",workspaceDir:null,startedAt:1,finishedAt:2,verdictToken:"FAILED",
|
|
failureReason:"test failure",nodeId:"test-node"
|
|
});`;
|
|
const proc = Bun.spawn([process.execPath, "-e", script], {
|
|
stdout: "pipe", stderr: "pipe", env: { ...process.env, SECURITY_TELEMETRY_DIR: stateDir },
|
|
});
|
|
const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
|
|
expect(code).toBe(1);
|
|
expect(stderr).toBe("");
|
|
expect(stdout).toContain("status-flushed");
|
|
const { readdirSync } = await import("node:fs");
|
|
expect(readdirSync(path.join(stateDir, "outbox")).length).toBe(3);
|
|
} finally { rmSync(stateDir, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test("API-key quota exhaustion does not trigger the three-failure process exit", async () => {
|
|
const stateDir = mkdtempSync(path.join(tmpdir(), "security-api-quota-exit-"));
|
|
try {
|
|
const script = `import { initSecurityFailureExitGuard } from ${JSON.stringify(modulePath)};
|
|
import { fireSecurityCallback } from ${JSON.stringify(path.join(root, "satellite/src/security_telemetry/callback.ts"))};
|
|
initSecurityFailureExitGuard(() => process.exit(91));
|
|
for (const id of ["a", "b", "quota"]) fireSecurityCallback({
|
|
event:"security_failed", security:{id,title:id,platform:"android",attempts:0},status:"security_failed",
|
|
phase:"run",workspaceDir:null,startedAt:1,finishedAt:2,verdictToken:"FAILED",
|
|
failureReason:"failure",nodeId:"test-node",
|
|
extra:id==="quota"?{failure_code:"AGENT_USAGE_LIMIT",provider:"cursor"}:null
|
|
});
|
|
setTimeout(() => console.log("satellite-still-running"), 10);`;
|
|
const proc = Bun.spawn([process.execPath, "-e", script], {
|
|
stdout: "pipe", stderr: "pipe",
|
|
env: { ...process.env, SECURITY_TELEMETRY_DIR: stateDir, STREAMLINE_CURSOR_AUTH_MODE: "api-key" },
|
|
});
|
|
const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
|
|
expect(code).toBe(0);
|
|
expect(stderr).toBe("");
|
|
expect(stdout).toContain("satellite-still-running");
|
|
} finally { rmSync(stateDir, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test("started, asking and manual cancellation never become execution failures", () => {
|
|
const source = readFileSync(path.join(root, "satellite/src/security_telemetry/callback.ts"), "utf8");
|
|
expect(source).toContain('opts.phase === "evaluate" || opts.phase === "run"');
|
|
expect(source).toContain('opts.event === "security_failed" || opts.event === "eval_ready" || opts.event === "manual_review"');
|
|
const evaluator = readFileSync(path.join(root, "satellite/src/security_evaluator.ts"), "utf8");
|
|
expect(evaluator).toContain("evaluationExecutionFailed = true");
|
|
expect(evaluator).toContain('outcome: evaluationExecutionFailed ? "failed" : "succeeded"');
|
|
expect(evaluator).toContain("if (!evalPool.evalAborted.has(security.id))");
|
|
});
|
|
|
|
test("only standalone satellite installs the process exit handler with a bounded flush", () => {
|
|
const standalone = readFileSync(path.join(root, "satellite/src/index.ts"), "utf8");
|
|
expect(standalone).toContain("initSecurityFailureExitGuard(async");
|
|
expect(standalone).toContain("setTimeout(exit, 5_000)");
|
|
expect(standalone).toContain("flushSecurityStatusWrites(), drainSecurityCallbackOutbox()");
|
|
expect(standalone).toContain("process.exit(1)");
|
|
expect(readFileSync(path.join(root, "control-server/src/index.ts"), "utf8")).not.toContain("initSecurityFailureExitGuard");
|
|
for (const file of ["feature_evaluator.ts", "feature_runner.ts", "runner.ts", "asking.ts"]) {
|
|
expect(readFileSync(path.join(root, "satellite/src", file), "utf8")).not.toContain("noteSecurityExecutionOutcome");
|
|
}
|
|
});
|
|
});
|