import { afterEach, describe, expect, test } from "bun:test"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { readSecurityUsageSnapshot, recordSecurityUsage, modelFromStreamJson, tokenUsageFromStreamJson, } from "../src/security_telemetry/usage"; import { DEFAULT_SECURITY_CALLBACK_URL, drainSecurityCallbackOutbox, enqueueSecurityCallbackPayload, enqueueSecurityCoverageUpload, } from "../src/security_telemetry/outbox"; import { buildSecurityCallbackPayload } from "../src/security_telemetry/callback"; import { securityCoverageArchiveForCallback, securityCoverageCallbackUrl, uploadSecurityCoverageArchive, } from "../src/security_telemetry/coverage"; const dirs: string[] = []; function stateDir(): string { const dir = mkdtempSync(path.join(tmpdir(), "streamline-security-telemetry-")); dirs.push(dir); return dir; } afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); function resultFrame(input: number, output: number, cacheRead = 0, cacheWrite = 0): Record { return { type: "result", usage: { input_tokens: input, output_tokens: output, cache_read_input_tokens: cacheRead, cache_creation_input_tokens: cacheWrite, }, }; } describe("security token usage ledger", () => { test("uses the terminal result frame instead of double-counting assistant frames", () => { const stream = [ JSON.stringify({ type: "assistant", message: { usage: { input_tokens: 999 } } }), JSON.stringify(resultFrame(11, 7, 5, 3)), ].join("\n"); expect(tokenUsageFromStreamJson(stream)).toEqual({ input_tokens: 11, output_tokens: 7, cache_read_input_tokens: 5, cache_creation_input_tokens: 3, thinking_tokens: 0, }); }); test("normalises provider-native reasoning and thinking token shapes into thinking_tokens", () => { const stream = [ JSON.stringify({ type: "system", subtype: "init", model: "gpt-5.6-sol" }), JSON.stringify({ type: "assistant", message: { id: "msg-1", model: "gpt-5.6-sol", usage: { output_tokens_details: { thinking_tokens: 9 } }, }, }), JSON.stringify({ type: "result", usage: { input_tokens: 21, output_tokens: 13, reasoning_output_tokens: 8, }, }), ].join("\n"); expect(modelFromStreamJson(stream)).toBe("gpt-5.6-sol"); expect(tokenUsageFromStreamJson(stream)).toEqual({ input_tokens: 21, output_tokens: 13, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, thinking_tokens: 9, }); }); test("keeps one cumulative snapshot across evaluate, run and asking", () => { const dir = stateDir(); recordSecurityUsage({ taskId: "sec-1", invocationId: "eval-1", stage: "evaluate", provider: "claude", model: "claude-opus-4-8", startedAt: 1, finishedAt: 2, resultFrame: resultFrame(100, 20, 10, 5), stateDir: dir, }); recordSecurityUsage({ taskId: "sec-1", invocationId: "run-1", stage: "run", provider: "codex", model: "gpt-5.6-sol", startedAt: 3, finishedAt: 4, streamJson: JSON.stringify(resultFrame(300, 60, 40, 0)), stateDir: dir, }); recordSecurityUsage({ taskId: "sec-1", invocationId: "asking-1", stage: "asking", provider: "cursor", model: "Opus 4.8 300K Max", startedAt: 5, finishedAt: 6, resultFrame: null, stateDir: dir, }); // Same invocation is idempotent, even if a replay carries different values. recordSecurityUsage({ taskId: "sec-1", invocationId: "run-1", stage: "run", provider: "codex", startedAt: 3, finishedAt: 7, resultFrame: resultFrame(999, 999), stateDir: dir, }); const snapshot = readSecurityUsageSnapshot("sec-1", dir); expect(snapshot.revision).toBe(3); expect(snapshot).not.toHaveProperty("provider"); expect(snapshot).not.toHaveProperty("model"); expect(snapshot.stages.evaluate).toMatchObject({ provider: "claude", model: "claude-opus-4-8", invocations: 1, input_tokens: 100, output_tokens: 20, }); expect(snapshot.stages.run).toMatchObject({ provider: "codex", model: "gpt-5.6-sol", invocations: 1, input_tokens: 300, output_tokens: 60, }); expect(snapshot.stages.asking).toMatchObject({ provider: "cursor", model: "Opus 4.8 300K Max", invocations: 1, incomplete_invocations: 1, }); expect(snapshot.total).toMatchObject({ invocations: 3, complete_invocations: 2, incomplete_invocations: 1, input_tokens: 400, output_tokens: 80, cache_read_input_tokens: 50, cache_creation_input_tokens: 5, thinking_tokens: 0, }); expect(snapshot.total).not.toHaveProperty("provider"); expect(snapshot.total).not.toHaveProperty("model"); }); test("every lifecycle callback carries the whole task snapshot", () => { const dir = stateDir(); recordSecurityUsage({ taskId: "sec-2", invocationId: "eval", stage: "evaluate", provider: "claude", model: "claude-sonnet", startedAt: 10, finishedAt: 20, resultFrame: resultFrame(30, 4), stateDir: dir, }); recordSecurityUsage({ taskId: "sec-2", invocationId: "run", stage: "run", provider: "claude", model: "claude-sonnet", startedAt: 21, finishedAt: 40, resultFrame: resultFrame(70, 8), stateDir: dir, }); const payload = buildSecurityCallbackPayload({ event: "manual_review", security: { id: "sec-2", title: "audit", platform: "android", attempts: 0 }, status: "manual_review", phase: "run", workspaceDir: null, startedAt: 21, finishedAt: 40, verdictToken: "EVIDENCE_READY", failureReason: null, nodeId: "sat-1", }, dir); expect(payload.usage_revision).toBe(2); expect(payload.token_usage).toMatchObject({ stages: { evaluate: { provider: "claude", model: "claude-sonnet", input_tokens: 30, output_tokens: 4 }, run: { provider: "claude", model: "claude-sonnet", input_tokens: 70, output_tokens: 8 }, }, total: { input_tokens: 100, output_tokens: 12 }, }); expect(payload.token_usage).not.toHaveProperty("provider"); expect(payload.token_usage).not.toHaveProperty("model"); expect((payload.token_usage as { total: Record }).total).not.toHaveProperty("provider"); expect((payload.token_usage as { total: Record }).total).not.toHaveProperty("model"); }); test("failure callbacks carry a structured extra reason", () => { const payload = buildSecurityCallbackPayload({ event: "security_failed", security: { id: "sec-failed", title: "audit", platform: "android", attempts: 0 }, status: "security_failed", phase: "run", workspaceDir: null, startedAt: 100, finishedAt: 200, verdictToken: "FAILED", failureReason: "AGENT_USAGE_LIMIT: usage exhausted", extra: { failure_code: "AGENT_USAGE_LIMIT", failure_reason: "usage exhausted", provider: "codex", retry_count: 3, max_retries: 3, }, nodeId: "sat-1", }, stateDir()); expect(payload.failure_reason).toBe("AGENT_USAGE_LIMIT: usage exhausted"); expect(payload.extra).toEqual({ failure_code: "AGENT_USAGE_LIMIT", failure_reason: "usage exhausted", provider: "codex", retry_count: 3, max_retries: 3, }); }); test("legacy security failures still mirror their reason into extra", () => { const payload = buildSecurityCallbackPayload({ event: "security_failed", security: { id: "sec-eval-failed", title: "audit", platform: "ios", attempts: 1 }, status: "security_failed", phase: "evaluate", workspaceDir: null, startedAt: 100, finishedAt: 200, verdictToken: "FAILED", failureReason: "evaluation gateway unavailable", nodeId: "sat-1", }, stateDir()); expect(payload.extra).toEqual({ failure_reason: "evaluation gateway unavailable" }); }); }); describe("security callback outbox", () => { test("uses the existing security callback URL by default", () => { expect(DEFAULT_SECURITY_CALLBACK_URL) .toBe("http://10.37.81.28:3100/api/workflow-callbacks/execution/streamline"); }); test("keeps a failed callback and delivers the same callback after restart", async () => { const dir = stateDir(); const callbackId = enqueueSecurityCallbackPayload( { event: "manual_review", task_id: "sec-3", node_id: "sat-1", token_usage: { revision: 1 } }, { stateDir: dir, now: () => 1_000 }, ); const seen: Array<{ id: string | null; body: string }> = []; const failingFetch = (async (_url: string | URL | Request, init?: RequestInit) => { const headers = new Headers(init?.headers); seen.push({ id: headers.get("x-streamline-callback-id"), body: String(init?.body || "") }); return new Response("down", { status: 503 }); }) as typeof fetch; await drainSecurityCallbackOutbox({ stateDir: dir, now: () => 1_000, fetchImpl: failingFetch }); const outbox = path.join(dir, "outbox"); expect(readdirSync(outbox).filter((name) => name.endsWith(".json"))).toHaveLength(1); const succeedingFetch = (async (_url: string | URL | Request, init?: RequestInit) => { const headers = new Headers(init?.headers); seen.push({ id: headers.get("x-streamline-callback-id"), body: String(init?.body || "") }); return new Response("", { status: 204 }); }) as typeof fetch; // A freshly-started dispatcher ignores the old backoff once. await drainSecurityCallbackOutbox({ stateDir: dir, now: () => 2_000, fetchImpl: succeedingFetch, forceDue: true, }); expect(readdirSync(outbox).filter((name) => name.endsWith(".json"))).toHaveLength(0); expect(seen.map((entry) => entry.id)).toEqual([callbackId, callbackId]); expect(seen[0]!.body).toBe(seen[1]!.body); expect(JSON.parse(seen[0]!.body).callback_id).toBe(callbackId); }); test("moves a non-retryable contract rejection to dead-letter", async () => { const dir = stateDir(); enqueueSecurityCallbackPayload({ event: "manual_review", task_id: "sec-4" }, { stateDir: dir }); const badRequest = (async () => new Response("bad event", { status: 400 })) as typeof fetch; await drainSecurityCallbackOutbox({ stateDir: dir, fetchImpl: badRequest, forceDue: true }); expect(readdirSync(path.join(dir, "outbox")).filter((name) => name.endsWith(".json"))).toHaveLength(0); const [dead] = readdirSync(path.join(dir, "dead-letter")); expect(JSON.parse(readFileSync(path.join(dir, "dead-letter", dead!), "utf8"))).toMatchObject({ event: "manual_review", last_error: "HTTP 400", }); }); test("only evidence-ready and failed callbacks select coverage.zip", () => { const dir = stateDir(); const archive = path.join(dir, "coverage.zip"); writeFileSync(archive, "zip"); expect(securityCoverageArchiveForCallback(dir, "EVIDENCE_READY")).toBe(archive); expect(securityCoverageArchiveForCallback(dir, "FAILED")).toBe(archive); expect(securityCoverageArchiveForCallback(dir, "UNREPRODUCIBLE")).toBeNull(); expect(securityCoverageArchiveForCallback(dir, null)).toBeNull(); expect(securityCoverageArchiveForCallback(null, "FAILED")).toBeNull(); }); test("uploads coverage.zip as multipart to the task coverage callback", async () => { const dir = stateDir(); const archive = path.join(dir, "coverage.zip"); writeFileSync(archive, "coverage-bytes"); const installedApk = path.join(dir, "installed.apk"); writeFileSync(installedApk, "apk-bytes"); enqueueSecurityCoverageUpload({ callbackId: "callback-coverage", taskId: "sec/coverage", nodeId: "sat-coverage", filePath: archive, }, { stateDir: dir, now: () => 3_000 }); const seen: Array<{ url: string; callbackId: string | null; contentType: string | null; bytes: string; }> = []; const fetchImpl = (async (url: string | URL | Request, init?: RequestInit) => { const headers = new Headers(init?.headers); const form = init?.body as FormData; const file = form.get("file") as File; seen.push({ url: String(url), callbackId: headers.get("x-streamline-callback-id"), contentType: headers.get("content-type"), bytes: await file.text(), }); return new Response("", { status: 204 }); }) as typeof fetch; const callbackUrl = "http://automation/api/workflow-callbacks/execution/streamline"; await drainSecurityCallbackOutbox({ stateDir: dir, callbackUrl, fetchImpl, forceDue: true, }); expect(seen).toEqual([{ url: securityCoverageCallbackUrl(callbackUrl, "sec/coverage"), callbackId: "callback-coverage", contentType: null, bytes: "coverage-bytes", }]); expect(readdirSync(path.join(dir, "coverage-outbox"))).toHaveLength(0); expect(existsSync(archive)).toBe(false); expect(existsSync(installedApk)).toBe(false); }); test("after coverage upload removes unchanged task APKs without following links", async () => { const dir = stateDir(); const workspace = path.join(dir, "security-task-1"); const nested = path.join(workspace, "artifacts", "prebuilt"); mkdirSync(nested, { recursive: true }); const archive = path.join(workspace, "coverage.zip"); const apk = path.join(nested, "app.APK"); const changed = path.join(nested, "changed.apk"); const outside = path.join(dir, "outside"); mkdirSync(outside); const other = path.join(outside, "other.apk"); writeFileSync(archive, "coverage"); writeFileSync(apk, "apk"); writeFileSync(changed, "old"); writeFileSync(other, "other"); symlinkSync(outside, path.join(workspace, "external")); enqueueSecurityCoverageUpload({ callbackId: "recursive-cleanup", taskId: "task-1", nodeId: "node", filePath: archive }, { stateDir: dir }); await drainSecurityCallbackOutbox({ stateDir: dir, forceDue: true, fetchImpl: (async () => { writeFileSync(changed, "replaced during upload"); return new Response(null, { status: 204 }); }) as typeof fetch }); expect(existsSync(archive)).toBe(false); expect(existsSync(apk)).toBe(false); expect(readFileSync(changed, "utf8")).toBe("replaced during upload"); expect(existsSync(other)).toBe(true); }); test("uses coverage.zip as the multipart wire filename", async () => { const dir = stateDir(); const archive = path.join(dir, "coverage.zip"); writeFileSync(archive, "coverage-wire-bytes"); let received: { pathname: string; contentType: string | null; fileName: string | null; fileSize: number | null; } | null = null; const server = Bun.serve({ hostname: "127.0.0.1", port: 0, async fetch(request) { const form = await request.formData(); const file = form.get("file"); received = { pathname: new URL(request.url).pathname, contentType: request.headers.get("content-type"), fileName: file instanceof File ? file.name : null, fileSize: file instanceof File ? file.size : null, }; return new Response(null, { status: 204 }); }, }); try { const result = await uploadSecurityCoverageArchive({ callbackUrl: `http://127.0.0.1:${server.port}/api/workflow-callbacks/execution/streamline`, taskId: "sec-wire", callbackId: "callback-wire", nodeId: "sat-wire", filePath: archive, }); expect(result).toEqual({ ok: true, retryable: false, error: "" }); expect(received).toEqual({ pathname: "/api/workflow-callbacks/execution/streamline/coverage/sec-wire", contentType: expect.stringContaining("multipart/form-data"), fileName: "coverage.zip", fileSize: "coverage-wire-bytes".length, }); } finally { server.stop(true); } }); test("keeps a failed coverage upload for retry without re-enqueuing JSON", async () => { const dir = stateDir(); const archive = path.join(dir, "coverage.zip"); writeFileSync(archive, "coverage-retry"); enqueueSecurityCoverageUpload({ callbackId: "callback-retry", taskId: "sec-retry", nodeId: "sat-retry", filePath: archive, }, { stateDir: dir, now: () => 4_000 }); const callbackIds: Array = []; const failingFetch = (async (_url: string | URL | Request, init?: RequestInit) => { callbackIds.push(new Headers(init?.headers).get("x-streamline-callback-id")); return new Response("down", { status: 503 }); }) as typeof fetch; await drainSecurityCallbackOutbox({ stateDir: dir, callbackUrl: "http://automation/api/workflow-callbacks/execution/streamline", fetchImpl: failingFetch, now: () => 4_000, forceDue: true, }); expect(readdirSync(path.join(dir, "coverage-outbox"))).toHaveLength(1); const succeedingFetch = (async (_url: string | URL | Request, init?: RequestInit) => { callbackIds.push(new Headers(init?.headers).get("x-streamline-callback-id")); return new Response("", { status: 204 }); }) as typeof fetch; await drainSecurityCallbackOutbox({ stateDir: dir, callbackUrl: "http://automation/api/workflow-callbacks/execution/streamline", fetchImpl: succeedingFetch, now: () => 5_000, forceDue: true, }); expect(readdirSync(path.join(dir, "coverage-outbox"))).toHaveLength(0); expect(callbackIds).toEqual(["callback-retry", "callback-retry"]); expect(existsSync(path.join(dir, "outbox"))).toBe(false); }); });