Files
streamline/satellite/test/security_telemetry.test.ts
2026-11-16 17:26:44 -08:00

453 lines
18 KiB
TypeScript

import { afterEach, describe, expect, test } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import {
readSecurityUsageSnapshot,
recordSecurityUsage,
modelFromStreamJson,
tokenUsageFromStreamJson,
} from "../src/security_telemetry/usage";
import {
DEFAULT_SECURITY_CALLBACK_URL,
drainSecurityCallbackOutbox,
enqueueSecurityCallbackPayload,
enqueueSecurityCoverageUpload,
} from "../src/security_telemetry/outbox";
import { buildSecurityCallbackPayload } from "../src/security_telemetry/callback";
import {
securityCoverageArchiveForCallback,
securityCoverageCallbackUrl,
uploadSecurityCoverageArchive,
} from "../src/security_telemetry/coverage";
const dirs: string[] = [];
function stateDir(): string {
const dir = mkdtempSync(path.join(tmpdir(), "streamline-security-telemetry-"));
dirs.push(dir);
return dir;
}
afterEach(() => {
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
function resultFrame(input: number, output: number, cacheRead = 0, cacheWrite = 0): Record<string, unknown> {
return {
type: "result",
usage: {
input_tokens: input,
output_tokens: output,
cache_read_input_tokens: cacheRead,
cache_creation_input_tokens: cacheWrite,
},
};
}
describe("security token usage ledger", () => {
test("uses the terminal result frame instead of double-counting assistant frames", () => {
const stream = [
JSON.stringify({ type: "assistant", message: { usage: { input_tokens: 999 } } }),
JSON.stringify(resultFrame(11, 7, 5, 3)),
].join("\n");
expect(tokenUsageFromStreamJson(stream)).toEqual({
input_tokens: 11,
output_tokens: 7,
cache_read_input_tokens: 5,
cache_creation_input_tokens: 3,
thinking_tokens: 0,
});
});
test("normalises provider-native reasoning and thinking token shapes into thinking_tokens", () => {
const stream = [
JSON.stringify({ type: "system", subtype: "init", model: "gpt-5.6-sol" }),
JSON.stringify({
type: "assistant",
message: {
id: "msg-1",
model: "gpt-5.6-sol",
usage: { output_tokens_details: { thinking_tokens: 9 } },
},
}),
JSON.stringify({
type: "result",
usage: {
input_tokens: 21,
output_tokens: 13,
reasoning_output_tokens: 8,
},
}),
].join("\n");
expect(modelFromStreamJson(stream)).toBe("gpt-5.6-sol");
expect(tokenUsageFromStreamJson(stream)).toEqual({
input_tokens: 21,
output_tokens: 13,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
thinking_tokens: 9,
});
});
test("keeps one cumulative snapshot across evaluate, run and asking", () => {
const dir = stateDir();
recordSecurityUsage({
taskId: "sec-1", invocationId: "eval-1", stage: "evaluate", provider: "claude",
model: "claude-opus-4-8", startedAt: 1, finishedAt: 2,
resultFrame: resultFrame(100, 20, 10, 5), stateDir: dir,
});
recordSecurityUsage({
taskId: "sec-1", invocationId: "run-1", stage: "run", provider: "codex",
model: "gpt-5.6-sol", startedAt: 3, finishedAt: 4,
streamJson: JSON.stringify(resultFrame(300, 60, 40, 0)), stateDir: dir,
});
recordSecurityUsage({
taskId: "sec-1", invocationId: "asking-1", stage: "asking", provider: "cursor",
model: "Opus 4.8 300K Max", startedAt: 5, finishedAt: 6, resultFrame: null, stateDir: dir,
});
// Same invocation is idempotent, even if a replay carries different values.
recordSecurityUsage({
taskId: "sec-1", invocationId: "run-1", stage: "run", provider: "codex",
startedAt: 3, finishedAt: 7, resultFrame: resultFrame(999, 999), stateDir: dir,
});
const snapshot = readSecurityUsageSnapshot("sec-1", dir);
expect(snapshot.revision).toBe(3);
expect(snapshot).not.toHaveProperty("provider");
expect(snapshot).not.toHaveProperty("model");
expect(snapshot.stages.evaluate).toMatchObject({
provider: "claude", model: "claude-opus-4-8", invocations: 1, input_tokens: 100, output_tokens: 20,
});
expect(snapshot.stages.run).toMatchObject({
provider: "codex", model: "gpt-5.6-sol", invocations: 1, input_tokens: 300, output_tokens: 60,
});
expect(snapshot.stages.asking).toMatchObject({
provider: "cursor", model: "Opus 4.8 300K Max", invocations: 1, incomplete_invocations: 1,
});
expect(snapshot.total).toMatchObject({
invocations: 3,
complete_invocations: 2,
incomplete_invocations: 1,
input_tokens: 400,
output_tokens: 80,
cache_read_input_tokens: 50,
cache_creation_input_tokens: 5,
thinking_tokens: 0,
});
expect(snapshot.total).not.toHaveProperty("provider");
expect(snapshot.total).not.toHaveProperty("model");
});
test("every lifecycle callback carries the whole task snapshot", () => {
const dir = stateDir();
recordSecurityUsage({
taskId: "sec-2", invocationId: "eval", stage: "evaluate", provider: "claude",
model: "claude-sonnet", startedAt: 10, finishedAt: 20,
resultFrame: resultFrame(30, 4), stateDir: dir,
});
recordSecurityUsage({
taskId: "sec-2", invocationId: "run", stage: "run", provider: "claude",
model: "claude-sonnet", startedAt: 21, finishedAt: 40,
resultFrame: resultFrame(70, 8), stateDir: dir,
});
const payload = buildSecurityCallbackPayload({
event: "manual_review",
security: { id: "sec-2", title: "audit", platform: "android", attempts: 0 },
status: "manual_review",
phase: "run",
workspaceDir: null,
startedAt: 21,
finishedAt: 40,
verdictToken: "EVIDENCE_READY",
failureReason: null,
nodeId: "sat-1",
}, dir);
expect(payload.usage_revision).toBe(2);
expect(payload.token_usage).toMatchObject({
stages: {
evaluate: { provider: "claude", model: "claude-sonnet", input_tokens: 30, output_tokens: 4 },
run: { provider: "claude", model: "claude-sonnet", input_tokens: 70, output_tokens: 8 },
},
total: { input_tokens: 100, output_tokens: 12 },
});
expect(payload.token_usage).not.toHaveProperty("provider");
expect(payload.token_usage).not.toHaveProperty("model");
expect((payload.token_usage as { total: Record<string, unknown> }).total).not.toHaveProperty("provider");
expect((payload.token_usage as { total: Record<string, unknown> }).total).not.toHaveProperty("model");
});
test("failure callbacks carry a structured extra reason", () => {
const payload = buildSecurityCallbackPayload({
event: "security_failed",
security: { id: "sec-failed", title: "audit", platform: "android", attempts: 0 },
status: "security_failed",
phase: "run",
workspaceDir: null,
startedAt: 100,
finishedAt: 200,
verdictToken: "FAILED",
failureReason: "AGENT_USAGE_LIMIT: usage exhausted",
extra: {
failure_code: "AGENT_USAGE_LIMIT",
failure_reason: "usage exhausted",
provider: "codex",
retry_count: 3,
max_retries: 3,
},
nodeId: "sat-1",
}, stateDir());
expect(payload.failure_reason).toBe("AGENT_USAGE_LIMIT: usage exhausted");
expect(payload.extra).toEqual({
failure_code: "AGENT_USAGE_LIMIT",
failure_reason: "usage exhausted",
provider: "codex",
retry_count: 3,
max_retries: 3,
});
});
test("legacy security failures still mirror their reason into extra", () => {
const payload = buildSecurityCallbackPayload({
event: "security_failed",
security: { id: "sec-eval-failed", title: "audit", platform: "ios", attempts: 1 },
status: "security_failed",
phase: "evaluate",
workspaceDir: null,
startedAt: 100,
finishedAt: 200,
verdictToken: "FAILED",
failureReason: "evaluation gateway unavailable",
nodeId: "sat-1",
}, stateDir());
expect(payload.extra).toEqual({ failure_reason: "evaluation gateway unavailable" });
});
});
describe("security callback outbox", () => {
test("uses the existing security callback URL by default", () => {
expect(DEFAULT_SECURITY_CALLBACK_URL)
.toBe("http://10.37.81.28:3100/api/workflow-callbacks/execution/streamline");
});
test("keeps a failed callback and delivers the same callback after restart", async () => {
const dir = stateDir();
const callbackId = enqueueSecurityCallbackPayload(
{ event: "manual_review", task_id: "sec-3", node_id: "sat-1", token_usage: { revision: 1 } },
{ stateDir: dir, now: () => 1_000 },
);
const seen: Array<{ id: string | null; body: string }> = [];
const failingFetch = (async (_url: string | URL | Request, init?: RequestInit) => {
const headers = new Headers(init?.headers);
seen.push({ id: headers.get("x-streamline-callback-id"), body: String(init?.body || "") });
return new Response("down", { status: 503 });
}) as typeof fetch;
await drainSecurityCallbackOutbox({ stateDir: dir, now: () => 1_000, fetchImpl: failingFetch });
const outbox = path.join(dir, "outbox");
expect(readdirSync(outbox).filter((name) => name.endsWith(".json"))).toHaveLength(1);
const succeedingFetch = (async (_url: string | URL | Request, init?: RequestInit) => {
const headers = new Headers(init?.headers);
seen.push({ id: headers.get("x-streamline-callback-id"), body: String(init?.body || "") });
return new Response("", { status: 204 });
}) as typeof fetch;
// A freshly-started dispatcher ignores the old backoff once.
await drainSecurityCallbackOutbox({
stateDir: dir, now: () => 2_000, fetchImpl: succeedingFetch, forceDue: true,
});
expect(readdirSync(outbox).filter((name) => name.endsWith(".json"))).toHaveLength(0);
expect(seen.map((entry) => entry.id)).toEqual([callbackId, callbackId]);
expect(seen[0]!.body).toBe(seen[1]!.body);
expect(JSON.parse(seen[0]!.body).callback_id).toBe(callbackId);
});
test("moves a non-retryable contract rejection to dead-letter", async () => {
const dir = stateDir();
enqueueSecurityCallbackPayload({ event: "manual_review", task_id: "sec-4" }, { stateDir: dir });
const badRequest = (async () => new Response("bad event", { status: 400 })) as typeof fetch;
await drainSecurityCallbackOutbox({ stateDir: dir, fetchImpl: badRequest, forceDue: true });
expect(readdirSync(path.join(dir, "outbox")).filter((name) => name.endsWith(".json"))).toHaveLength(0);
const [dead] = readdirSync(path.join(dir, "dead-letter"));
expect(JSON.parse(readFileSync(path.join(dir, "dead-letter", dead!), "utf8"))).toMatchObject({
event: "manual_review",
last_error: "HTTP 400",
});
});
test("only evidence-ready and failed callbacks select coverage.zip", () => {
const dir = stateDir();
const archive = path.join(dir, "coverage.zip");
writeFileSync(archive, "zip");
expect(securityCoverageArchiveForCallback(dir, "EVIDENCE_READY")).toBe(archive);
expect(securityCoverageArchiveForCallback(dir, "FAILED")).toBe(archive);
expect(securityCoverageArchiveForCallback(dir, "UNREPRODUCIBLE")).toBeNull();
expect(securityCoverageArchiveForCallback(dir, null)).toBeNull();
expect(securityCoverageArchiveForCallback(null, "FAILED")).toBeNull();
});
test("uploads coverage.zip as multipart to the task coverage callback", async () => {
const dir = stateDir();
const archive = path.join(dir, "coverage.zip");
writeFileSync(archive, "coverage-bytes");
const installedApk = path.join(dir, "installed.apk");
writeFileSync(installedApk, "apk-bytes");
enqueueSecurityCoverageUpload({
callbackId: "callback-coverage",
taskId: "sec/coverage",
nodeId: "sat-coverage",
filePath: archive,
}, { stateDir: dir, now: () => 3_000 });
const seen: Array<{
url: string;
callbackId: string | null;
contentType: string | null;
bytes: string;
}> = [];
const fetchImpl = (async (url: string | URL | Request, init?: RequestInit) => {
const headers = new Headers(init?.headers);
const form = init?.body as FormData;
const file = form.get("file") as File;
seen.push({
url: String(url),
callbackId: headers.get("x-streamline-callback-id"),
contentType: headers.get("content-type"),
bytes: await file.text(),
});
return new Response("", { status: 204 });
}) as typeof fetch;
const callbackUrl = "http://automation/api/workflow-callbacks/execution/streamline";
await drainSecurityCallbackOutbox({
stateDir: dir,
callbackUrl,
fetchImpl,
forceDue: true,
});
expect(seen).toEqual([{
url: securityCoverageCallbackUrl(callbackUrl, "sec/coverage"),
callbackId: "callback-coverage",
contentType: null,
bytes: "coverage-bytes",
}]);
expect(readdirSync(path.join(dir, "coverage-outbox"))).toHaveLength(0);
expect(existsSync(archive)).toBe(false);
expect(existsSync(installedApk)).toBe(false);
});
test("after coverage upload removes unchanged task APKs without following links", async () => {
const dir = stateDir();
const workspace = path.join(dir, "security-task-1");
const nested = path.join(workspace, "artifacts", "prebuilt");
mkdirSync(nested, { recursive: true });
const archive = path.join(workspace, "coverage.zip");
const apk = path.join(nested, "app.APK");
const changed = path.join(nested, "changed.apk");
const outside = path.join(dir, "outside");
mkdirSync(outside);
const other = path.join(outside, "other.apk");
writeFileSync(archive, "coverage"); writeFileSync(apk, "apk");
writeFileSync(changed, "old"); writeFileSync(other, "other");
symlinkSync(outside, path.join(workspace, "external"));
enqueueSecurityCoverageUpload({ callbackId: "recursive-cleanup", taskId: "task-1", nodeId: "node", filePath: archive }, { stateDir: dir });
await drainSecurityCallbackOutbox({ stateDir: dir, forceDue: true, fetchImpl: (async () => {
writeFileSync(changed, "replaced during upload");
return new Response(null, { status: 204 });
}) as typeof fetch });
expect(existsSync(archive)).toBe(false);
expect(existsSync(apk)).toBe(false);
expect(readFileSync(changed, "utf8")).toBe("replaced during upload");
expect(existsSync(other)).toBe(true);
});
test("uses coverage.zip as the multipart wire filename", async () => {
const dir = stateDir();
const archive = path.join(dir, "coverage.zip");
writeFileSync(archive, "coverage-wire-bytes");
let received: {
pathname: string;
contentType: string | null;
fileName: string | null;
fileSize: number | null;
} | null = null;
const server = Bun.serve({
hostname: "127.0.0.1",
port: 0,
async fetch(request) {
const form = await request.formData();
const file = form.get("file");
received = {
pathname: new URL(request.url).pathname,
contentType: request.headers.get("content-type"),
fileName: file instanceof File ? file.name : null,
fileSize: file instanceof File ? file.size : null,
};
return new Response(null, { status: 204 });
},
});
try {
const result = await uploadSecurityCoverageArchive({
callbackUrl: `http://127.0.0.1:${server.port}/api/workflow-callbacks/execution/streamline`,
taskId: "sec-wire",
callbackId: "callback-wire",
nodeId: "sat-wire",
filePath: archive,
});
expect(result).toEqual({ ok: true, retryable: false, error: "" });
expect(received).toEqual({
pathname: "/api/workflow-callbacks/execution/streamline/coverage/sec-wire",
contentType: expect.stringContaining("multipart/form-data"),
fileName: "coverage.zip",
fileSize: "coverage-wire-bytes".length,
});
} finally {
server.stop(true);
}
});
test("keeps a failed coverage upload for retry without re-enqueuing JSON", async () => {
const dir = stateDir();
const archive = path.join(dir, "coverage.zip");
writeFileSync(archive, "coverage-retry");
enqueueSecurityCoverageUpload({
callbackId: "callback-retry",
taskId: "sec-retry",
nodeId: "sat-retry",
filePath: archive,
}, { stateDir: dir, now: () => 4_000 });
const callbackIds: Array<string | null> = [];
const failingFetch = (async (_url: string | URL | Request, init?: RequestInit) => {
callbackIds.push(new Headers(init?.headers).get("x-streamline-callback-id"));
return new Response("down", { status: 503 });
}) as typeof fetch;
await drainSecurityCallbackOutbox({
stateDir: dir,
callbackUrl: "http://automation/api/workflow-callbacks/execution/streamline",
fetchImpl: failingFetch,
now: () => 4_000,
forceDue: true,
});
expect(readdirSync(path.join(dir, "coverage-outbox"))).toHaveLength(1);
const succeedingFetch = (async (_url: string | URL | Request, init?: RequestInit) => {
callbackIds.push(new Headers(init?.headers).get("x-streamline-callback-id"));
return new Response("", { status: 204 });
}) as typeof fetch;
await drainSecurityCallbackOutbox({
stateDir: dir,
callbackUrl: "http://automation/api/workflow-callbacks/execution/streamline",
fetchImpl: succeedingFetch,
now: () => 5_000,
forceDue: true,
});
expect(readdirSync(path.join(dir, "coverage-outbox"))).toHaveLength(0);
expect(callbackIds).toEqual(["callback-retry", "callback-retry"]);
expect(existsSync(path.join(dir, "outbox"))).toBe(false);
});
});