Files
streamline/satellite/test/security_failure_streak.test.ts
T
2026-11-16 17:26:44 -08:00

140 lines
8.0 KiB
TypeScript

import { describe, expect, test } from "bun:test";
import { readFileSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { SecurityFailureStreak, type SecurityExecutionOutcome } from "../src/security_telemetry/failure_streak";
const root = path.resolve(import.meta.dir, "../..");
const modulePath = path.join(root, "satellite/src/security_telemetry/failure_streak.ts");
function outcome(taskId: string, phase: "evaluate" | "run" = "run", result: "failed" | "succeeded" = "failed", startedAt = 1): SecurityExecutionOutcome {
return { taskId, phase, outcome: result, startedAt };
}
describe("Security failed-task exit guard", () => {
test("evaluate and run share a three-distinct-task threshold", () => {
const streak = new SecurityFailureStreak();
expect(streak.record(outcome("a", "evaluate"))).toBe(false);
expect(streak.record(outcome("b", "run"))).toBe(false);
expect(streak.record(outcome("c", "evaluate"))).toBe(true);
expect(streak.snapshot()).toMatchObject({ exitRequested: true, failedTasks: [
{ taskId: "a", phase: "evaluate" }, { taskId: "b", phase: "run" }, { taskId: "c", phase: "evaluate" },
] });
expect(streak.record(outcome("d"))).toBe(false);
});
test("repeated failures and both stages of one task still count once", () => {
const streak = new SecurityFailureStreak();
streak.record(outcome("a", "evaluate"));
streak.record(outcome("a", "evaluate", "failed", 2));
streak.record(outcome("a", "run"));
expect(streak.snapshot().failedTasks.length).toBe(1);
expect(streak.record(outcome("b"))).toBe(false);
expect(streak.record(outcome("c"))).toBe(true);
});
test("a successful stage clears the streak and duplicate success cannot clear later failures", () => {
const streak = new SecurityFailureStreak();
streak.record(outcome("a"));
streak.record(outcome("b"));
streak.record(outcome("ready", "evaluate", "succeeded"));
expect(streak.snapshot().failedTasks).toEqual([]);
streak.record(outcome("c"));
streak.record(outcome("ready", "evaluate", "succeeded"));
expect(streak.snapshot().failedTasks.length).toBe(1);
expect(streak.record(outcome("d"))).toBe(false);
expect(streak.record(outcome("e"))).toBe(true);
});
test("success after the third failure cannot rescind the requested exit", () => {
const streak = new SecurityFailureStreak();
for (const id of ["a", "b", "c"]) streak.record(outcome(id));
streak.record(outcome("ready", "run", "succeeded"));
expect(streak.snapshot().exitRequested).toBe(true);
expect(new SecurityFailureStreak().snapshot().failedTasks).toEqual([]);
});
test("inactive control-plane imports never request process exit", async () => {
const script = `import { noteSecurityExecutionOutcome } from ${JSON.stringify(modulePath)};
for (const taskId of ["a", "b", "c"]) noteSecurityExecutionOutcome({ taskId, phase:"run", startedAt:1, outcome:"failed" });
console.log("control-plane-survived");`;
const proc = Bun.spawn([process.execPath, "-e", script], { stdout: "pipe", stderr: "pipe" });
const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
expect(code).toBe(0);
expect(stderr).toBe("");
expect(stdout).toContain("control-plane-survived");
});
test("third failure callback is durable before exit and status writes can finish", async () => {
const stateDir = mkdtempSync(path.join(tmpdir(), "security-failure-exit-"));
try {
const script = `import { initSecurityFailureExitGuard, trackSecurityStatusWrite, flushSecurityStatusWrites } from ${JSON.stringify(modulePath)};
import { fireSecurityCallback } from ${JSON.stringify(path.join(root, "satellite/src/security_telemetry/callback.ts"))};
initSecurityFailureExitGuard(async () => {
await flushSecurityStatusWrites(); console.log("status-flushed"); process.exit(1);
});
trackSecurityStatusWrite(new Promise(resolve => setTimeout(resolve, 30)));
for (const id of ["a", "b", "c"]) fireSecurityCallback({
event:"security_failed", security:{id,title:id,platform:"android",attempts:0},status:"security_failed",
phase:id==="b"?"evaluate":"run",workspaceDir:null,startedAt:1,finishedAt:2,verdictToken:"FAILED",
failureReason:"test failure",nodeId:"test-node"
});`;
const proc = Bun.spawn([process.execPath, "-e", script], {
stdout: "pipe", stderr: "pipe", env: { ...process.env, SECURITY_TELEMETRY_DIR: stateDir },
});
const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
expect(code).toBe(1);
expect(stderr).toBe("");
expect(stdout).toContain("status-flushed");
const { readdirSync } = await import("node:fs");
expect(readdirSync(path.join(stateDir, "outbox")).length).toBe(3);
} finally { rmSync(stateDir, { recursive: true, force: true }); }
});
test("API-key quota exhaustion does not trigger the three-failure process exit", async () => {
const stateDir = mkdtempSync(path.join(tmpdir(), "security-api-quota-exit-"));
try {
const script = `import { initSecurityFailureExitGuard } from ${JSON.stringify(modulePath)};
import { fireSecurityCallback } from ${JSON.stringify(path.join(root, "satellite/src/security_telemetry/callback.ts"))};
initSecurityFailureExitGuard(() => process.exit(91));
for (const id of ["a", "b", "quota"]) fireSecurityCallback({
event:"security_failed", security:{id,title:id,platform:"android",attempts:0},status:"security_failed",
phase:"run",workspaceDir:null,startedAt:1,finishedAt:2,verdictToken:"FAILED",
failureReason:"failure",nodeId:"test-node",
extra:id==="quota"?{failure_code:"AGENT_USAGE_LIMIT",provider:"cursor"}:null
});
setTimeout(() => console.log("satellite-still-running"), 10);`;
const proc = Bun.spawn([process.execPath, "-e", script], {
stdout: "pipe", stderr: "pipe",
env: { ...process.env, SECURITY_TELEMETRY_DIR: stateDir, STREAMLINE_CURSOR_AUTH_MODE: "api-key" },
});
const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
expect(code).toBe(0);
expect(stderr).toBe("");
expect(stdout).toContain("satellite-still-running");
} finally { rmSync(stateDir, { recursive: true, force: true }); }
});
test("started, asking and manual cancellation never become execution failures", () => {
const source = readFileSync(path.join(root, "satellite/src/security_telemetry/callback.ts"), "utf8");
expect(source).toContain('opts.phase === "evaluate" || opts.phase === "run"');
expect(source).toContain('opts.event === "security_failed" || opts.event === "eval_ready" || opts.event === "manual_review"');
const evaluator = readFileSync(path.join(root, "satellite/src/security_evaluator.ts"), "utf8");
expect(evaluator).toContain("evaluationExecutionFailed = true");
expect(evaluator).toContain('outcome: evaluationExecutionFailed ? "failed" : "succeeded"');
expect(evaluator).toContain("if (!evalPool.evalAborted.has(security.id))");
});
test("only standalone satellite installs the process exit handler with a bounded flush", () => {
const standalone = readFileSync(path.join(root, "satellite/src/index.ts"), "utf8");
expect(standalone).toContain("initSecurityFailureExitGuard(async");
expect(standalone).toContain("setTimeout(exit, 5_000)");
expect(standalone).toContain("flushSecurityStatusWrites(), drainSecurityCallbackOutbox()");
expect(standalone).toContain("process.exit(1)");
expect(readFileSync(path.join(root, "control-server/src/index.ts"), "utf8")).not.toContain("initSecurityFailureExitGuard");
for (const file of ["feature_evaluator.ts", "feature_runner.ts", "runner.ts", "asking.ts"]) {
expect(readFileSync(path.join(root, "satellite/src", file), "utf8")).not.toContain("noteSecurityExecutionOutcome");
}
});
});