Initial import of streamline
This commit is contained in:
@@ -0,0 +1,452 @@
|
||||
import { afterEach, describe, expect, test } from "bun:test";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
readSecurityUsageSnapshot,
|
||||
recordSecurityUsage,
|
||||
modelFromStreamJson,
|
||||
tokenUsageFromStreamJson,
|
||||
} from "../src/security_telemetry/usage";
|
||||
import {
|
||||
DEFAULT_SECURITY_CALLBACK_URL,
|
||||
drainSecurityCallbackOutbox,
|
||||
enqueueSecurityCallbackPayload,
|
||||
enqueueSecurityCoverageUpload,
|
||||
} from "../src/security_telemetry/outbox";
|
||||
import { buildSecurityCallbackPayload } from "../src/security_telemetry/callback";
|
||||
import {
|
||||
securityCoverageArchiveForCallback,
|
||||
securityCoverageCallbackUrl,
|
||||
uploadSecurityCoverageArchive,
|
||||
} from "../src/security_telemetry/coverage";
|
||||
|
||||
const dirs: string[] = [];
|
||||
|
||||
function stateDir(): string {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "streamline-security-telemetry-"));
|
||||
dirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function resultFrame(input: number, output: number, cacheRead = 0, cacheWrite = 0): Record<string, unknown> {
|
||||
return {
|
||||
type: "result",
|
||||
usage: {
|
||||
input_tokens: input,
|
||||
output_tokens: output,
|
||||
cache_read_input_tokens: cacheRead,
|
||||
cache_creation_input_tokens: cacheWrite,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("security token usage ledger", () => {
|
||||
test("uses the terminal result frame instead of double-counting assistant frames", () => {
|
||||
const stream = [
|
||||
JSON.stringify({ type: "assistant", message: { usage: { input_tokens: 999 } } }),
|
||||
JSON.stringify(resultFrame(11, 7, 5, 3)),
|
||||
].join("\n");
|
||||
expect(tokenUsageFromStreamJson(stream)).toEqual({
|
||||
input_tokens: 11,
|
||||
output_tokens: 7,
|
||||
cache_read_input_tokens: 5,
|
||||
cache_creation_input_tokens: 3,
|
||||
thinking_tokens: 0,
|
||||
});
|
||||
});
|
||||
|
||||
test("normalises provider-native reasoning and thinking token shapes into thinking_tokens", () => {
|
||||
const stream = [
|
||||
JSON.stringify({ type: "system", subtype: "init", model: "gpt-5.6-sol" }),
|
||||
JSON.stringify({
|
||||
type: "assistant",
|
||||
message: {
|
||||
id: "msg-1",
|
||||
model: "gpt-5.6-sol",
|
||||
usage: { output_tokens_details: { thinking_tokens: 9 } },
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "result",
|
||||
usage: {
|
||||
input_tokens: 21,
|
||||
output_tokens: 13,
|
||||
reasoning_output_tokens: 8,
|
||||
},
|
||||
}),
|
||||
].join("\n");
|
||||
expect(modelFromStreamJson(stream)).toBe("gpt-5.6-sol");
|
||||
expect(tokenUsageFromStreamJson(stream)).toEqual({
|
||||
input_tokens: 21,
|
||||
output_tokens: 13,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
thinking_tokens: 9,
|
||||
});
|
||||
});
|
||||
|
||||
test("keeps one cumulative snapshot across evaluate, run and asking", () => {
|
||||
const dir = stateDir();
|
||||
recordSecurityUsage({
|
||||
taskId: "sec-1", invocationId: "eval-1", stage: "evaluate", provider: "claude",
|
||||
model: "claude-opus-4-8", startedAt: 1, finishedAt: 2,
|
||||
resultFrame: resultFrame(100, 20, 10, 5), stateDir: dir,
|
||||
});
|
||||
recordSecurityUsage({
|
||||
taskId: "sec-1", invocationId: "run-1", stage: "run", provider: "codex",
|
||||
model: "gpt-5.6-sol", startedAt: 3, finishedAt: 4,
|
||||
streamJson: JSON.stringify(resultFrame(300, 60, 40, 0)), stateDir: dir,
|
||||
});
|
||||
recordSecurityUsage({
|
||||
taskId: "sec-1", invocationId: "asking-1", stage: "asking", provider: "cursor",
|
||||
model: "Opus 4.8 300K Max", startedAt: 5, finishedAt: 6, resultFrame: null, stateDir: dir,
|
||||
});
|
||||
// Same invocation is idempotent, even if a replay carries different values.
|
||||
recordSecurityUsage({
|
||||
taskId: "sec-1", invocationId: "run-1", stage: "run", provider: "codex",
|
||||
startedAt: 3, finishedAt: 7, resultFrame: resultFrame(999, 999), stateDir: dir,
|
||||
});
|
||||
|
||||
const snapshot = readSecurityUsageSnapshot("sec-1", dir);
|
||||
expect(snapshot.revision).toBe(3);
|
||||
expect(snapshot).not.toHaveProperty("provider");
|
||||
expect(snapshot).not.toHaveProperty("model");
|
||||
expect(snapshot.stages.evaluate).toMatchObject({
|
||||
provider: "claude", model: "claude-opus-4-8", invocations: 1, input_tokens: 100, output_tokens: 20,
|
||||
});
|
||||
expect(snapshot.stages.run).toMatchObject({
|
||||
provider: "codex", model: "gpt-5.6-sol", invocations: 1, input_tokens: 300, output_tokens: 60,
|
||||
});
|
||||
expect(snapshot.stages.asking).toMatchObject({
|
||||
provider: "cursor", model: "Opus 4.8 300K Max", invocations: 1, incomplete_invocations: 1,
|
||||
});
|
||||
expect(snapshot.total).toMatchObject({
|
||||
invocations: 3,
|
||||
complete_invocations: 2,
|
||||
incomplete_invocations: 1,
|
||||
input_tokens: 400,
|
||||
output_tokens: 80,
|
||||
cache_read_input_tokens: 50,
|
||||
cache_creation_input_tokens: 5,
|
||||
thinking_tokens: 0,
|
||||
});
|
||||
expect(snapshot.total).not.toHaveProperty("provider");
|
||||
expect(snapshot.total).not.toHaveProperty("model");
|
||||
});
|
||||
|
||||
test("every lifecycle callback carries the whole task snapshot", () => {
|
||||
const dir = stateDir();
|
||||
recordSecurityUsage({
|
||||
taskId: "sec-2", invocationId: "eval", stage: "evaluate", provider: "claude",
|
||||
model: "claude-sonnet", startedAt: 10, finishedAt: 20,
|
||||
resultFrame: resultFrame(30, 4), stateDir: dir,
|
||||
});
|
||||
recordSecurityUsage({
|
||||
taskId: "sec-2", invocationId: "run", stage: "run", provider: "claude",
|
||||
model: "claude-sonnet", startedAt: 21, finishedAt: 40,
|
||||
resultFrame: resultFrame(70, 8), stateDir: dir,
|
||||
});
|
||||
const payload = buildSecurityCallbackPayload({
|
||||
event: "manual_review",
|
||||
security: { id: "sec-2", title: "audit", platform: "android", attempts: 0 },
|
||||
status: "manual_review",
|
||||
phase: "run",
|
||||
workspaceDir: null,
|
||||
startedAt: 21,
|
||||
finishedAt: 40,
|
||||
verdictToken: "EVIDENCE_READY",
|
||||
failureReason: null,
|
||||
nodeId: "sat-1",
|
||||
}, dir);
|
||||
expect(payload.usage_revision).toBe(2);
|
||||
expect(payload.token_usage).toMatchObject({
|
||||
stages: {
|
||||
evaluate: { provider: "claude", model: "claude-sonnet", input_tokens: 30, output_tokens: 4 },
|
||||
run: { provider: "claude", model: "claude-sonnet", input_tokens: 70, output_tokens: 8 },
|
||||
},
|
||||
total: { input_tokens: 100, output_tokens: 12 },
|
||||
});
|
||||
expect(payload.token_usage).not.toHaveProperty("provider");
|
||||
expect(payload.token_usage).not.toHaveProperty("model");
|
||||
expect((payload.token_usage as { total: Record<string, unknown> }).total).not.toHaveProperty("provider");
|
||||
expect((payload.token_usage as { total: Record<string, unknown> }).total).not.toHaveProperty("model");
|
||||
});
|
||||
|
||||
test("failure callbacks carry a structured extra reason", () => {
|
||||
const payload = buildSecurityCallbackPayload({
|
||||
event: "security_failed",
|
||||
security: { id: "sec-failed", title: "audit", platform: "android", attempts: 0 },
|
||||
status: "security_failed",
|
||||
phase: "run",
|
||||
workspaceDir: null,
|
||||
startedAt: 100,
|
||||
finishedAt: 200,
|
||||
verdictToken: "FAILED",
|
||||
failureReason: "AGENT_USAGE_LIMIT: usage exhausted",
|
||||
extra: {
|
||||
failure_code: "AGENT_USAGE_LIMIT",
|
||||
failure_reason: "usage exhausted",
|
||||
provider: "codex",
|
||||
retry_count: 3,
|
||||
max_retries: 3,
|
||||
},
|
||||
nodeId: "sat-1",
|
||||
}, stateDir());
|
||||
|
||||
expect(payload.failure_reason).toBe("AGENT_USAGE_LIMIT: usage exhausted");
|
||||
expect(payload.extra).toEqual({
|
||||
failure_code: "AGENT_USAGE_LIMIT",
|
||||
failure_reason: "usage exhausted",
|
||||
provider: "codex",
|
||||
retry_count: 3,
|
||||
max_retries: 3,
|
||||
});
|
||||
});
|
||||
|
||||
test("legacy security failures still mirror their reason into extra", () => {
|
||||
const payload = buildSecurityCallbackPayload({
|
||||
event: "security_failed",
|
||||
security: { id: "sec-eval-failed", title: "audit", platform: "ios", attempts: 1 },
|
||||
status: "security_failed",
|
||||
phase: "evaluate",
|
||||
workspaceDir: null,
|
||||
startedAt: 100,
|
||||
finishedAt: 200,
|
||||
verdictToken: "FAILED",
|
||||
failureReason: "evaluation gateway unavailable",
|
||||
nodeId: "sat-1",
|
||||
}, stateDir());
|
||||
|
||||
expect(payload.extra).toEqual({ failure_reason: "evaluation gateway unavailable" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("security callback outbox", () => {
|
||||
test("uses the existing security callback URL by default", () => {
|
||||
expect(DEFAULT_SECURITY_CALLBACK_URL)
|
||||
.toBe("http://10.37.81.28:3100/api/workflow-callbacks/execution/streamline");
|
||||
});
|
||||
|
||||
test("keeps a failed callback and delivers the same callback after restart", async () => {
|
||||
const dir = stateDir();
|
||||
const callbackId = enqueueSecurityCallbackPayload(
|
||||
{ event: "manual_review", task_id: "sec-3", node_id: "sat-1", token_usage: { revision: 1 } },
|
||||
{ stateDir: dir, now: () => 1_000 },
|
||||
);
|
||||
const seen: Array<{ id: string | null; body: string }> = [];
|
||||
const failingFetch = (async (_url: string | URL | Request, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers);
|
||||
seen.push({ id: headers.get("x-streamline-callback-id"), body: String(init?.body || "") });
|
||||
return new Response("down", { status: 503 });
|
||||
}) as typeof fetch;
|
||||
await drainSecurityCallbackOutbox({ stateDir: dir, now: () => 1_000, fetchImpl: failingFetch });
|
||||
const outbox = path.join(dir, "outbox");
|
||||
expect(readdirSync(outbox).filter((name) => name.endsWith(".json"))).toHaveLength(1);
|
||||
|
||||
const succeedingFetch = (async (_url: string | URL | Request, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers);
|
||||
seen.push({ id: headers.get("x-streamline-callback-id"), body: String(init?.body || "") });
|
||||
return new Response("", { status: 204 });
|
||||
}) as typeof fetch;
|
||||
// A freshly-started dispatcher ignores the old backoff once.
|
||||
await drainSecurityCallbackOutbox({
|
||||
stateDir: dir, now: () => 2_000, fetchImpl: succeedingFetch, forceDue: true,
|
||||
});
|
||||
expect(readdirSync(outbox).filter((name) => name.endsWith(".json"))).toHaveLength(0);
|
||||
expect(seen.map((entry) => entry.id)).toEqual([callbackId, callbackId]);
|
||||
expect(seen[0]!.body).toBe(seen[1]!.body);
|
||||
expect(JSON.parse(seen[0]!.body).callback_id).toBe(callbackId);
|
||||
});
|
||||
|
||||
test("moves a non-retryable contract rejection to dead-letter", async () => {
|
||||
const dir = stateDir();
|
||||
enqueueSecurityCallbackPayload({ event: "manual_review", task_id: "sec-4" }, { stateDir: dir });
|
||||
const badRequest = (async () => new Response("bad event", { status: 400 })) as typeof fetch;
|
||||
await drainSecurityCallbackOutbox({ stateDir: dir, fetchImpl: badRequest, forceDue: true });
|
||||
expect(readdirSync(path.join(dir, "outbox")).filter((name) => name.endsWith(".json"))).toHaveLength(0);
|
||||
const [dead] = readdirSync(path.join(dir, "dead-letter"));
|
||||
expect(JSON.parse(readFileSync(path.join(dir, "dead-letter", dead!), "utf8"))).toMatchObject({
|
||||
event: "manual_review",
|
||||
last_error: "HTTP 400",
|
||||
});
|
||||
});
|
||||
|
||||
test("only evidence-ready and failed callbacks select coverage.zip", () => {
|
||||
const dir = stateDir();
|
||||
const archive = path.join(dir, "coverage.zip");
|
||||
writeFileSync(archive, "zip");
|
||||
|
||||
expect(securityCoverageArchiveForCallback(dir, "EVIDENCE_READY")).toBe(archive);
|
||||
expect(securityCoverageArchiveForCallback(dir, "FAILED")).toBe(archive);
|
||||
expect(securityCoverageArchiveForCallback(dir, "UNREPRODUCIBLE")).toBeNull();
|
||||
expect(securityCoverageArchiveForCallback(dir, null)).toBeNull();
|
||||
expect(securityCoverageArchiveForCallback(null, "FAILED")).toBeNull();
|
||||
});
|
||||
|
||||
test("uploads coverage.zip as multipart to the task coverage callback", async () => {
|
||||
const dir = stateDir();
|
||||
const archive = path.join(dir, "coverage.zip");
|
||||
writeFileSync(archive, "coverage-bytes");
|
||||
const installedApk = path.join(dir, "installed.apk");
|
||||
writeFileSync(installedApk, "apk-bytes");
|
||||
enqueueSecurityCoverageUpload({
|
||||
callbackId: "callback-coverage",
|
||||
taskId: "sec/coverage",
|
||||
nodeId: "sat-coverage",
|
||||
filePath: archive,
|
||||
}, { stateDir: dir, now: () => 3_000 });
|
||||
|
||||
const seen: Array<{
|
||||
url: string;
|
||||
callbackId: string | null;
|
||||
contentType: string | null;
|
||||
bytes: string;
|
||||
}> = [];
|
||||
const fetchImpl = (async (url: string | URL | Request, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers);
|
||||
const form = init?.body as FormData;
|
||||
const file = form.get("file") as File;
|
||||
seen.push({
|
||||
url: String(url),
|
||||
callbackId: headers.get("x-streamline-callback-id"),
|
||||
contentType: headers.get("content-type"),
|
||||
bytes: await file.text(),
|
||||
});
|
||||
return new Response("", { status: 204 });
|
||||
}) as typeof fetch;
|
||||
|
||||
const callbackUrl = "http://automation/api/workflow-callbacks/execution/streamline";
|
||||
await drainSecurityCallbackOutbox({
|
||||
stateDir: dir,
|
||||
callbackUrl,
|
||||
fetchImpl,
|
||||
forceDue: true,
|
||||
});
|
||||
|
||||
expect(seen).toEqual([{
|
||||
url: securityCoverageCallbackUrl(callbackUrl, "sec/coverage"),
|
||||
callbackId: "callback-coverage",
|
||||
contentType: null,
|
||||
bytes: "coverage-bytes",
|
||||
}]);
|
||||
expect(readdirSync(path.join(dir, "coverage-outbox"))).toHaveLength(0);
|
||||
expect(existsSync(archive)).toBe(false);
|
||||
expect(existsSync(installedApk)).toBe(false);
|
||||
});
|
||||
|
||||
test("after coverage upload removes unchanged task APKs without following links", async () => {
|
||||
const dir = stateDir();
|
||||
const workspace = path.join(dir, "security-task-1");
|
||||
const nested = path.join(workspace, "artifacts", "prebuilt");
|
||||
mkdirSync(nested, { recursive: true });
|
||||
const archive = path.join(workspace, "coverage.zip");
|
||||
const apk = path.join(nested, "app.APK");
|
||||
const changed = path.join(nested, "changed.apk");
|
||||
const outside = path.join(dir, "outside");
|
||||
mkdirSync(outside);
|
||||
const other = path.join(outside, "other.apk");
|
||||
writeFileSync(archive, "coverage"); writeFileSync(apk, "apk");
|
||||
writeFileSync(changed, "old"); writeFileSync(other, "other");
|
||||
symlinkSync(outside, path.join(workspace, "external"));
|
||||
enqueueSecurityCoverageUpload({ callbackId: "recursive-cleanup", taskId: "task-1", nodeId: "node", filePath: archive }, { stateDir: dir });
|
||||
await drainSecurityCallbackOutbox({ stateDir: dir, forceDue: true, fetchImpl: (async () => {
|
||||
writeFileSync(changed, "replaced during upload");
|
||||
return new Response(null, { status: 204 });
|
||||
}) as typeof fetch });
|
||||
expect(existsSync(archive)).toBe(false);
|
||||
expect(existsSync(apk)).toBe(false);
|
||||
expect(readFileSync(changed, "utf8")).toBe("replaced during upload");
|
||||
expect(existsSync(other)).toBe(true);
|
||||
});
|
||||
|
||||
test("uses coverage.zip as the multipart wire filename", async () => {
|
||||
const dir = stateDir();
|
||||
const archive = path.join(dir, "coverage.zip");
|
||||
writeFileSync(archive, "coverage-wire-bytes");
|
||||
let received: {
|
||||
pathname: string;
|
||||
contentType: string | null;
|
||||
fileName: string | null;
|
||||
fileSize: number | null;
|
||||
} | null = null;
|
||||
const server = Bun.serve({
|
||||
hostname: "127.0.0.1",
|
||||
port: 0,
|
||||
async fetch(request) {
|
||||
const form = await request.formData();
|
||||
const file = form.get("file");
|
||||
received = {
|
||||
pathname: new URL(request.url).pathname,
|
||||
contentType: request.headers.get("content-type"),
|
||||
fileName: file instanceof File ? file.name : null,
|
||||
fileSize: file instanceof File ? file.size : null,
|
||||
};
|
||||
return new Response(null, { status: 204 });
|
||||
},
|
||||
});
|
||||
try {
|
||||
const result = await uploadSecurityCoverageArchive({
|
||||
callbackUrl: `http://127.0.0.1:${server.port}/api/workflow-callbacks/execution/streamline`,
|
||||
taskId: "sec-wire",
|
||||
callbackId: "callback-wire",
|
||||
nodeId: "sat-wire",
|
||||
filePath: archive,
|
||||
});
|
||||
expect(result).toEqual({ ok: true, retryable: false, error: "" });
|
||||
expect(received).toEqual({
|
||||
pathname: "/api/workflow-callbacks/execution/streamline/coverage/sec-wire",
|
||||
contentType: expect.stringContaining("multipart/form-data"),
|
||||
fileName: "coverage.zip",
|
||||
fileSize: "coverage-wire-bytes".length,
|
||||
});
|
||||
} finally {
|
||||
server.stop(true);
|
||||
}
|
||||
});
|
||||
|
||||
test("keeps a failed coverage upload for retry without re-enqueuing JSON", async () => {
|
||||
const dir = stateDir();
|
||||
const archive = path.join(dir, "coverage.zip");
|
||||
writeFileSync(archive, "coverage-retry");
|
||||
enqueueSecurityCoverageUpload({
|
||||
callbackId: "callback-retry",
|
||||
taskId: "sec-retry",
|
||||
nodeId: "sat-retry",
|
||||
filePath: archive,
|
||||
}, { stateDir: dir, now: () => 4_000 });
|
||||
|
||||
const callbackIds: Array<string | null> = [];
|
||||
const failingFetch = (async (_url: string | URL | Request, init?: RequestInit) => {
|
||||
callbackIds.push(new Headers(init?.headers).get("x-streamline-callback-id"));
|
||||
return new Response("down", { status: 503 });
|
||||
}) as typeof fetch;
|
||||
await drainSecurityCallbackOutbox({
|
||||
stateDir: dir,
|
||||
callbackUrl: "http://automation/api/workflow-callbacks/execution/streamline",
|
||||
fetchImpl: failingFetch,
|
||||
now: () => 4_000,
|
||||
forceDue: true,
|
||||
});
|
||||
expect(readdirSync(path.join(dir, "coverage-outbox"))).toHaveLength(1);
|
||||
|
||||
const succeedingFetch = (async (_url: string | URL | Request, init?: RequestInit) => {
|
||||
callbackIds.push(new Headers(init?.headers).get("x-streamline-callback-id"));
|
||||
return new Response("", { status: 204 });
|
||||
}) as typeof fetch;
|
||||
await drainSecurityCallbackOutbox({
|
||||
stateDir: dir,
|
||||
callbackUrl: "http://automation/api/workflow-callbacks/execution/streamline",
|
||||
fetchImpl: succeedingFetch,
|
||||
now: () => 5_000,
|
||||
forceDue: true,
|
||||
});
|
||||
expect(readdirSync(path.join(dir, "coverage-outbox"))).toHaveLength(0);
|
||||
expect(callbackIds).toEqual(["callback-retry", "callback-retry"]);
|
||||
expect(existsSync(path.join(dir, "outbox"))).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user