Files
streamline/control-server/test/eddy_usage.integration.test.ts
2026-11-16 17:26:44 -08:00

273 lines
22 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import './setup_env';
import { afterAll, expect, mock, test } from 'bun:test';
import { createHash } from 'node:crypto';
mock.module('../src/cluster', () => ({allEnrichedNodes: async () => [], callNodeAbsolute: async () => {throw new Error('No runtime execution');}}));
mock.module('../src/sso', () => ({resolveSsoEmail: async (h: Headers) => h.get('x-tt-session-v2') === 'usage-admin' ? 'usage-admin@example.com' : h.get('x-tt-session-v2') === 'usage-user' ? 'usage-user@example.com' : null}));
mock.module('../src/admin_db', () => ({isAdminPrefix: async (value: string) => value === 'usage-admin', normPrefix: (value: string) => value.split('@')[0]}));
const run = process.env.TEST_DATABASE_URL ? test : test.skip;
const { eddyApi } = await import('../src/routes_eddy');
const { db, closePool } = await import('../src/db_client');
const { eddy_usage_events, eddy_sso_sessions } = await import('../src/schema');
const { eq, inArray, sql } = await import('drizzle-orm');
const installations = ['a'.repeat(32), 'b'.repeat(32), 'e'.repeat(32)];
const token = 'eddy_sso_' + 'u'.repeat(43), tokenHash = createHash('sha256').update(token).digest('hex');
const user = {'x-tt-session-v2': 'usage-user'}, admin = {'x-tt-session-v2': 'usage-admin'};
const id = (n: number) => n.toString(16).padStart(32, '0');
const batch = (events: any[]) => ({schema: 1, batchId: 'f'.repeat(64), events});
const event = (i: number, changes = {}) => ({id: id(i), installation: installations[0], at: Date.now() - 5000, name: 'operation.completed', action: 'task.save',
source: 'portal', release: '2026.11.03.1', macosMajor: '27', outcome: 'success', durationMs: 100, ...changes});
const postOnce = (events: any[], headers = user) => eddyApi.request('/usage', {method: 'POST', headers: {'Content-Type': 'application/json', ...headers}, body: JSON.stringify(batch(events))});
const post = async (events: any[], headers = user) => {
for (let i = 0; ; i++) {
const response = await postOnce(events, headers);
if (i >= 20 || response.status !== 429 || (await response.clone().json() as any).code !== 'eddy_rate_limited') return response;
await Bun.sleep(20);
}
};
const get = (headers = admin, query = '') => eddyApi.request('/admin/usage' + query, {headers});
run('usage requires verified SSO; metrics are aggregate-only and administrator-only', async () => {
expect((await post([event(1)], {})).status).toBe(401);
expect((await post([event(1)], {'x-eddy-user': 'usage-admin'} as any)).status).toBe(401);
expect((await get({})).status).toBe(401);
expect((await get(user)).status).toBe(403);
await db.insert(eddy_sso_sessions).values({tokenHash, email: 'usage-admin@example.com', expiresAt: Date.now() + 60_000});
const headers = {authorization: 'Bearer ' + token};
expect((await post([event(1)], headers as any)).status).toBe(200);
expect((await get(headers as any)).status).toBe(401); // Even the admin's client session has no admin rights.
const response = await get(); expect(response.status).toBe(200); expect(response.headers.get('cache-control')).toBe('no-store');
const data: any = await response.json();
expect(data.summary.activeInstallations).toBe(1);
expect(JSON.stringify(data)).not.toContain(installations[0]);
// Per-user rows name the uploading account (administrators see who uses Eddy), never installation ids.
expect(data.users.find((u: any) => u.user === 'usage-admin@example.com').activeInstallations).toBe(1);
});
run('idempotent replay, failures, cancellation and percentiles have exact denominators', async () => {
const events = [event(2, {durationMs: 300}), event(3, {outcome: 'failure', durationMs: 500}), event(4, {outcome: 'cancelled', durationMs: 700}),
event(5, {name: 'operation.started', durationMs: undefined, outcome: undefined}),
event(6, {name: 'engagement', action: 'portal', durationMs: 30000, outcome: undefined, session: id(1)}),
event(7, {name: 'session.started', action: 'portal', durationMs: undefined, outcome: undefined, session: id(1)}),
event(8, {name: 'view.opened', action: 'home', outcome: undefined, durationMs: undefined, session: id(1), installation: installations[1], release: 'older'}),
event(9, {name: 'app.launch', action: 'app', source: 'runtime', durationMs: 800})];
for (const response of await Promise.all([post(events), post(events)])) {
expect(response.status).toBe(200); expect(await response.json()).toEqual({batchId: 'f'.repeat(64), accepted: events.length});
}
const data: any = await (await get()).json();
expect(data.summary.activeInstallations).toBe(2); expect(data.summary.portalSessions).toBe(1); expect(data.summary.portalActiveMs).toBe(30000);
const row = data.operations.find((r: any) => r.name === 'operation.completed' && r.action === 'task.save');
expect(row.count).toBe(4); expect(row.succeeded).toBe(2); expect(row.failed).toBe(1); expect(row.cancelled).toBe(1);
expect(row.successRate).toBeCloseTo(2 / 3); expect(row.p50Ms).toBe(200); expect(row.p95Ms).toBeCloseTo(290);
expect(data.daily.reduce((n: number, r: any) => n + r.actions, 0)).toBe(1);
const filtered: any = await (await get(admin, '?days=7&release=older')).json();
expect(filtered.summary.activeInstallations).toBe(1); expect(filtered.operations.length).toBe(1);
expect((await get(admin, '?days=91')).status).toBe(400);
});
run('unknown fields and large payloads are rejected without storing raw data', async () => {
expect((await post([event(10, {text: 'PRIVATE DATA'})])).status).toBe(400);
expect((await post([event(10, {release: 'x'.repeat(140 * 1024)})])).status).toBe(413);
const stored = await db.select().from(eddy_usage_events);
expect(JSON.stringify(stored)).not.toContain('PRIVATE DATA');
expect(stored.length).toBe(9);
});
run('raw fixed-schema events older than 90 days since receipt are removed', async () => {
const identifier = 'c'.repeat(64);
await db.insert(eddy_usage_events).values({id: identifier, at: Date.now() - 91 * 86400_000, receivedAt: Date.now() - 91 * 86400_000,
installation: installations[0], name: 'view.opened', action: 'home', source: 'portal', release: 'old', macosMajor: '26'});
const {pruneUsage} = await import('../src/eddy_usage'); await pruneUsage(true);
expect(await db.select().from(eddy_usage_events).where(eq(eddy_usage_events.id, identifier))).toEqual([]);
});
const runEddy = process.env.TEST_DATABASE_URL && process.env.EDDY_TEST_SOURCE ? test : test.skip;
runEddy('real Eddy queues fixed events and uploads idempotently through HTTP into Postgres', async () => {
const source = process.env.EDDY_TEST_SOURCE!;
const server = Bun.serve({hostname: '127.0.0.1', port: 0, fetch: request => {
const url = new URL(request.url); url.pathname = url.pathname.replace(/^\/api\/eddy/, '');
return eddyApi.fetch(new Request(url, request));
}});
try {
const child = Bun.spawn([source + '/vendor/python/bin/python3', source + '/tests/usage_server_probe.py', `http://127.0.0.1:${server.port}`, token], {stdout:'pipe',stderr:'pipe'});
const [exit, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
expect(stderr).toBe(''); expect(exit).toBe(0); expect(JSON.parse(stdout).ok).toBe(true);
const data: any = await (await get(admin, '?release=probe')).json();
expect(data.summary.activeInstallations).toBe(1);
const row = data.operations.find((r: any) => r.name === 'operation.completed');
expect(row.count).toBe(2); expect(row.failed).toBe(1); expect(row.successRate).toBe(.5);
expect(data.summary.portalActiveMs).toBe(30000);
// The signed-out Mac app event: same device, no account, counted per device.
expect(data.summary.activeDevices).toBe(1); expect(data.summary.anonymousDevices).toBe(1);
expect(data.operations.find((r: any) => r.name === 'home.shown').source).toBe('shell');
expect(data.users.map((u: any) => u.user).sort()).toEqual(['anonymous', 'usage-admin@example.com']);
} finally {server.stop(true);}
}, 20000);
run('ingestion monitoring is SSO-admin-only and omits per-owner identifiers', async () => {
const get = (headers: Record<string,string>)=>eddyApi.request('/admin/ingestion',{headers});
expect((await get({})).status).toBe(401); expect((await get(user)).status).toBe(403);
const response=await get(admin); expect(response.status).toBe(200);
const data:any=await response.json(); expect(data.channels.length).toBe(2);
expect(JSON.stringify(data)).not.toContain('usage-user@example.com');
});
run('per-user rows group by account; earlier hash-only events match known accounts; user filters every section', async () => {
const sha = (s: string) => createHash('sha256').update(s).digest('hex'), release = 'per-user-test';
expect((await post([event(101, {installation: installations[2], release}),
event(102, {name: 'app.launch', action: 'app', source: 'runtime', outcome: 'failure', installation: installations[2], release})])).status).toBe(200);
const [stored] = await db.select().from(eddy_usage_events).where(eq(eddy_usage_events.id, sha(installations[2] + ':' + id(101))));
expect(stored.owner).toBe('usage-user@example.com');
const legacy = (n: number, budgetOwner: string | null) => ({id: sha('legacy' + n), at: Date.now() - 4000, receivedAt: Date.now(), installation: installations[2],
name: 'operation.started', action: 'task.save', source: 'portal', release, macosMajor: '27', budgetOwner, ingestionBytes: 1});
await db.insert(eddy_usage_events).values([legacy(1, sha('usage-user@example.com')), legacy(2, sha('ghost@example.com')), legacy(3, null)]);
const data: any = await (await get(admin, '?release=' + release)).json();
const byUser = new Map(data.users.map((u: any) => [u.user, u]));
expect([...byUser.keys()].sort()).toEqual([null, 'sha256:' + sha('ghost@example.com').slice(0, 12), 'usage-user@example.com'].sort() as any);
expect((byUser.get('usage-user@example.com') as any).actions).toBe(1);
expect((byUser.get('usage-user@example.com') as any).launchesFailed).toBe(1);
const one: any = await (await get(admin, `?release=${release}&user=Usage-User@example.com`)).json();
expect(one.user).toBe('usage-user@example.com');
expect(one.users.map((u: any) => u.user)).toEqual(['usage-user@example.com']);
expect(one.operations.find((r: any) => r.name === 'app.launch').failed).toBe(1);
expect(one.daily.reduce((n: number, r: any) => n + r.actions, 0)).toBe(1); // the hash-matched earlier start
expect(one.summary.activeInstallations).toBe(1);
expect((await get(admin, '?user=not-an-email')).status).toBe(400);
await db.delete(eddy_usage_events).where(inArray(eddy_usage_events.id, [1, 2, 3].map(n => sha('legacy' + n))));
});
// Schema 2: device, production only, uploads without a sign-in charged to the device.
const deviceA = '0123456789abcdef', deviceB = 'fedcba9876543210', v2Installs = ['d'.repeat(32), '9'.repeat(32)];
const v2 = (i: number, changes: Record<string, unknown> = {}) => ({...event(i, {installation: v2Installs[0], release: 'devices-test', source: 'shell'}),
device: deviceA, deviceSource: 'hardware', env: 'production', appVersion: '0.9.13+15', ...changes});
const shellEvent = (i: number, name: string, action: string, changes: Record<string, unknown> = {}) =>
v2(i, {name, action, outcome: undefined, durationMs: undefined, session: id(900), ...changes});
const postV2 = (events: any[], headers: Record<string, string> = {}) => eddyApi.request('/usage', {method: 'POST',
headers: {'Content-Type': 'application/json', ...headers}, body: JSON.stringify({schema: 2, batchId: 'e'.repeat(64), events})});
run('schema 2 is accepted without a sign-in, per device, production only; a bad credential is still refused', async () => {
const sha = (s: string) => createHash('sha256').update(s).digest('hex');
expect((await postV2([v2(201, {release: 'anonymous-test'})])).status).toBe(200);
const [row] = await db.select().from(eddy_usage_events).where(eq(eddy_usage_events.id, sha(v2Installs[0] + ':' + id(201))));
expect(row.owner).toBeNull(); expect(row.anonymous).toBe(true); expect(row.device).toBe(deviceA); expect(row.appVersion).toBe('0.9.13+15');
expect(row.budgetOwner).toBe(sha('anonymous')); // one shared budget, whatever device id the caller sends
expect((await post([event(202)], {})).status).toBe(401); // schema 1 still needs a sign-in
expect((await postV2([v2(203, {release: 'anonymous-test'}), v2(204, {device: deviceB, release: 'anonymous-test'})])).status).toBe(400); // one device per anonymous batch
expect((await postV2([v2(205, {env: 'test', release: 'anonymous-test'})])).status).toBe(400);
expect((await postV2([v2(206, {release: 'anonymous-test'})], {authorization: 'Bearer eddy_sso_' + 'x'.repeat(43)})).status).toBe(401);
expect((await postV2([v2(207, {release: 'anonymous-test'})], {authorization: 'Bearer ' + token})).status).toBe(200); // signed in: the account's
const [signed] = await db.select().from(eddy_usage_events).where(eq(eddy_usage_events.id, sha(v2Installs[0] + ':' + id(207))));
expect(signed.owner).toBe('usage-admin@example.com'); expect(signed.anonymous).toBe(false);
});
run('a new device id buys no new budget: every unsigned upload is charged to one key and adds no budget rows', async () => {
const keys = async () => (await db.execute(sql`SELECT count(*)::int AS n FROM eddy_ingestion_budget WHERE key LIKE 'usage:owner:%'`)).rows[0] as any;
await postV2([v2(250, {release: 'anonymous-test'})]); // the shared budget exists from here on
const before = (await keys()).n;
for (let i = 0; i < 5; i++) {
const device = (0x1000 + i).toString(16).padStart(16, 'c');
expect((await postV2([v2(260 + i, {release: 'anonymous-test', device})])).status).toBe(200);
}
expect((await keys()).n).toBe(before);
});
run('devices, Mac app sessions, activation, adoption and flows; per-device rows never name accounts', async () => {
const now = Date.now(), at = (minutes: number) => now - 60 * 60_000 + minutes * 60_000;
const first = [
shellEvent(301, 'session.started', 'shell', {at: at(0), props: {firstLaunch: true, reason: 'manual'}}),
shellEvent(302, 'runtime.prepare', 'first-launch', {at: at(1), outcome: 'success', durationMs: 60000, props: {downloadMs: 41000, setupMs: 19000}}),
shellEvent(303, 'home.shown', 'shell', {at: at(2), durationMs: 64000, props: {firstLaunch: true}}),
shellEvent(304, 'engagement', 'shell', {at: at(3), durationMs: 240000}),
shellEvent(305, 'view.opened', 'desk', {at: at(4)}),
shellEvent(306, 'device.profile', 'shell', {at: at(4), props: {model: 'Mac15,6', chip: 'm3', memoryGB: 36, macos: '26.6', language: 'zh', location: 'applications'}}),
shellEvent(307, 'permission.flow', 'screen-recording', {at: at(5), props: {step: 'granted'}}),
v2(308, {at: at(10), name: 'operation.completed', action: 'app.install', outcome: 'success', durationMs: 90000}),
v2(309, {at: at(12), name: 'app.launch', action: 'app', source: 'runtime', outcome: 'success', durationMs: 30000, appVersion: undefined}),
v2(310, {at: at(20), name: 'operation.completed', action: 'task.export', outcome: 'success', durationMs: 2000, props: {reuse: 'fresh', media: true}}),
];
expect((await postV2(first)).status).toBe(200);
// A second Mac, signed in, that was there before (no first launch): active but not in the funnel.
expect((await postV2([v2(311, {device: deviceB, installation: v2Installs[1], at: at(30), name: 'app.launch', action: 'app', source: 'runtime', outcome: 'failure', durationMs: 90000}),
shellEvent(312, 'update.flow', 'app', {device: deviceB, installation: v2Installs[1], at: at(31), props: {step: 'applied', from: '0.9.12+14', to: '0.9.13+15', waitedMs: 5000}})],
{authorization: 'Bearer ' + token})).status).toBe(200);
const data: any = await (await get(admin, '?days=7&release=devices-test')).json();
expect(data.schema).toBe(2);
expect(data.summary.activeDevices).toBe(2); expect(data.summary.anonymousDevices).toBe(1); expect(data.summary.activeAccounts).toBe(1);
expect(data.summary.shellSessions).toBe(1); expect(data.summary.shellActiveMs).toBe(240000); expect(data.summary.portalSessions).toBe(0);
expect(data.funnel.firstLaunch).toBe(1); expect(data.funnel.runtimeReady).toBe(1); expect(data.funnel.homeShown).toBe(1);
expect(data.funnel.appInstalled).toBe(1); expect(data.funnel.taskOpened).toBe(1); expect(data.funnel.firstEdit).toBe(0); expect(data.funnel.firstShare).toBe(1);
expect(data.funnel.taskOpenedP50Ms).toBe(12 * 60_000);
expect(data.adoption.activeDevices).toBe(2); expect(data.adoption.openedTask).toBe(1); expect(data.adoption.designDesk).toBe(1); expect(data.adoption.share).toBe(1);
const devices = new Map(data.devices.map((d: any) => [d.device, d]));
const a: any = devices.get(deviceA.slice(0, 8)), b: any = devices.get(deviceB.slice(0, 8));
expect(a.neverSignedIn).toBe(true); expect(a.profile.chip).toBe('m3'); expect(a.appVersions).toEqual(['0.9.13+15']); expect(a.launchesSucceeded).toBe(1);
expect(b.neverSignedIn).toBe(false); expect(b.accounts).toBe(1); expect(b.launchesFailed).toBe(1);
expect(data.users.find((u: any) => u.user === 'anonymous').activeDevices).toBe(1);
expect(data.flows).toContainEqual({name: 'update.flow', action: 'app', step: 'applied', count: 1, devices: 1});
expect(data.flows).toContainEqual({name: 'permission.flow', action: 'screen-recording', step: 'granted', count: 1, devices: 1});
expect(data.appVersions.map((v: any) => v.appVersion)).toEqual(['0.9.13+15']);
expect(data.operations.find((r: any) => r.name === 'runtime.prepare').succeeded).toBe(1);
// Per-device rows: the first 8 hex digits only, never full ids, installations or account names.
const rows = JSON.stringify(data.devices);
for (const raw of [deviceA, deviceB, ...v2Installs, 'usage-admin@example.com']) expect(rows).not.toContain(raw);
});
run('P1 detail: desk, previews, docking, Agent entry, failure cards, crashes and storage cleaning by action and kind', async () => {
const p1 = (i: number, changes: Record<string, unknown>) => v2(i, {release: 'p1-test', device: 'abababababababab', ...changes});
const sh = (i: number, name: string, action: string, changes: Record<string, unknown> = {}) =>
p1(i, {name, action, source: 'shell', outcome: undefined, durationMs: undefined, session: id(960), ...changes});
expect((await postV2([
sh(401, 'desk.action', 'screenshot', {outcome: 'success'}), sh(402, 'desk.action', 'record', {outcome: 'success', durationMs: 42000}),
sh(403, 'preview.change', 'language', {outcome: 'success', durationMs: 20000, props: {value: 'ar'}}),
sh(404, 'preview.change', 'language', {outcome: 'failure', durationMs: 30000, props: {value: 'ar'}}),
sh(405, 'dock.attach', 'attached'), sh(406, 'dock.attach', 'detached', {durationMs: 600000}),
sh(407, 'agent.entry', 'app', {props: {agent: 'claude', resumed: false, element: true}}),
sh(408, 'notice.shown', 'failure', {props: {kind: 'space'}}), sh(409, 'notice.action', 'clean-up', {props: {kind: 'space'}}),
p1(410, {name: 'operation.completed', action: 'storage.clean', outcome: 'success', durationMs: 900, props: {what: 'app-cache', freedMB: 1200}}),
p1(411, {name: 'operation.completed', action: 'storage.clean', outcome: 'success', durationMs: 700, props: {what: 'app-cache', freedMB: 300}}),
p1(412, {name: 'app.crash', action: 'app', source: 'runtime', outcome: undefined, durationMs: undefined, appVersion: undefined, props: {class: 'devicesim', restart: 'restarted'}}),
p1(413, {name: 'app.session', action: 'app', source: 'runtime', outcome: 'success', durationMs: 1_800_000, appVersion: undefined}),
])).status).toBe(200);
const data: any = await (await get(admin, '?days=7&release=p1-test')).json();
const row = (name: string, action: string, detail = '') => data.details.find((r: any) => r.name === name && r.action === action && r.detail === detail);
expect(row('preview.change', 'language', 'ar')).toMatchObject({count: 2, devices: 1, succeeded: 1, failed: 1, p50Ms: 25000});
expect(row('desk.action', 'record')).toMatchObject({count: 1, succeeded: 1, p50Ms: 42000});
expect(row('dock.attach', 'detached').p50Ms).toBe(600000);
expect(row('agent.entry', 'app', 'claude').count).toBe(1);
expect(row('notice.action', 'clean-up', 'space').count).toBe(1);
expect(row('operation.completed', 'storage.clean', 'app-cache')).toMatchObject({count: 2, freedMB: 1500});
expect(row('app.crash', 'app', 'devicesim').count).toBe(1);
expect(data.summary.appCrashes).toBe(1); expect(data.summary.appRuntimeMs).toBe(1_800_000);
expect(data.adoption).toMatchObject({activeDevices: 1, deskCapture: 1, preview: 1, dock: 1, agentEntry: 1});
});
run('retention counts a window only once it has passed', async () => {
const sha = (s: string) => createHash('sha256').update(s).digest('hex'), day = 86400_000, base = Date.now() - 40 * day;
const row = (n: number, at: number, extra: Record<string, unknown> = {}) => ({id: sha('retention' + n), at, receivedAt: Date.now(), installation: v2Installs[1],
name: 'view.opened', action: 'home', source: 'shell', release: 'retention-test', macosMajor: '26', session: id(950), device: 'aaaaaaaaaaaaaaaa',
deviceSource: 'hardware', anonymous: true, ingestionBytes: 1, ...extra});
await db.insert(eddy_usage_events).values([
row(1, base, {name: 'session.started', action: 'shell', props: {firstLaunch: true}}), row(2, base + day + 3600_000), row(3, base + 8 * day),
// a device that started 3 days ago: day 1 has passed, days 7–13 and 28–34 have not
row(4, Date.now() - 3 * day, {device: 'bbbbbbbbbbbbbbbb', name: 'session.started', action: 'shell', props: {firstLaunch: true}}),
]);
const data: any = await (await get(admin, '?days=90&release=retention-test')).json();
const total = (key: string) => data.retention.reduce((n: number, r: any) => n + r[key], 0);
expect(total('devices')).toBe(2);
expect(total('d1Eligible')).toBe(2); expect(total('d1')).toBe(1);
expect(total('d7Eligible')).toBe(1); expect(total('d7')).toBe(1);
expect(total('d28Eligible')).toBe(1); expect(total('d28')).toBe(0);
await db.delete(eddy_usage_events).where(inArray(eddy_usage_events.id, [1, 2, 3, 4].map(n => sha('retention' + n))));
});
afterAll(async () => {
if (process.env.TEST_DATABASE_URL) await db.delete(eddy_usage_events).where(inArray(eddy_usage_events.installation, v2Installs));
});
afterAll(async () => {
if (process.env.TEST_DATABASE_URL) {
await db.delete(eddy_usage_events).where(inArray(eddy_usage_events.installation, installations));
await db.delete(eddy_sso_sessions).where(eq(eddy_sso_sessions.tokenHash, tokenHash));
}
await closePool();
});