Files
streamline/control-server/test/eddy_ingestion.integration.test.ts
2026-11-16 17:26:44 -08:00

111 lines
9.0 KiB
TypeScript

import './setup_env';
import {expect, test, beforeAll, afterAll} from 'bun:test';
import {createHash} from 'node:crypto';
import {sql} from 'drizzle-orm';
import {db, closePool} from '../src/db_client';
import {putConversation, getConversations, pruneConversations} from '../src/eddy_conversations';
import {conversationQuery} from '../src/eddy_conversation_rules';
import {admit, ownerKey, policy, IngestionDenied} from '../src/eddy_ingestion';
const run = process.env.TEST_DATABASE_URL ? test : test.skip;
const runTag = Math.random().toString(36).slice(2,9);
const email = (name: string) => `${name}-${runTag}@example.com`;
const hash = (s: string) => createHash('sha256').update(s).digest('hex');
process.env.EDDY_CONVERSATION_GLOBAL_BYTES = String(100 * 1024 ** 2);
process.env.EDDY_USAGE_GLOBAL_BYTES = String(100 * 1024 ** 2);
process.env.EDDY_CONVERSATION_OWNER_BURST = '10000';
process.env.EDDY_CONVERSATION_GLOBAL_BURST = '10000';
const batch = (name = 'one') => ({schema: 1 as const, batchId: hash(name), taskId: 'limit-a', taskTitle: 'Fixture',
sourceId: hash('limit-source'), provider: 'codex' as const, machine: 'f'.repeat(16),
events: [{id: hash(name), position: 1, block: 0, part: 0, parts: 1, kind: 'message' as const, role: 'user' as const, text: '同一可见事件🧪'.repeat(20), redacted: false}]});
const totals = async () => (await db.execute(sql`SELECT
(SELECT count(*) FROM eddy_conversation_records) + (SELECT count(*) FROM eddy_conversation_events) AS rows,
(SELECT coalesce(sum(payload_bytes),0) FROM eddy_conversation_records) + (SELECT coalesce(sum(ingestion_bytes),0) FROM eddy_conversation_events) AS bytes`)).rows[0] as any;
beforeAll(async () => {
if (!process.env.TEST_DATABASE_URL) return;
const t = await totals();
await db.execute(sql`INSERT INTO eddy_ingestion_budget (key,bytes,rows) VALUES ('conversation:global',${Number(t.bytes)},${Number(t.rows)}) ON CONFLICT (key) DO UPDATE SET bytes=excluded.bytes,rows=excluded.rows`);
});
const budget = async (key: string) => (await db.execute(sql`SELECT * FROM eddy_ingestion_budget WHERE key = ${key}`)).rows[0] as any;
run('migration preserves two task histories while storing their body once and seeds budgets', async () => {
const old = await Bun.file(new URL('../../drizzle/0065_eddy_conversations.sql',import.meta.url)).text();
const usage = await Bun.file(new URL('../../drizzle/0066_eddy_usage.sql',import.meta.url)).text();
const migration = await Bun.file(new URL('../../drizzle/0067_eddy_ingestion_budgets.sql',import.meta.url)).text();
await db.transaction(async tx => {
await tx.execute(sql`CREATE SCHEMA eddy_migration_fixture`);
await tx.execute(sql`SET LOCAL search_path = eddy_migration_fixture`);
await tx.execute(sql.raw(old)); await tx.execute(sql.raw(usage));
const event=JSON.stringify(batch().events[0]);
await tx.execute(sql`INSERT INTO eddy_conversation_events (id,owner,machine,task_id,task_title,source_id,provider,event,received_at)
SELECT task,'migration@example.com',repeat('f',16),task,task,repeat('a',64),'codex',${event},1 FROM unnest(ARRAY['a','b']) task`);
await tx.execute(sql.raw(migration)); await tx.execute(sql.raw(migration)); // Replay is idempotent.
const r=await tx.execute(sql`SELECT (SELECT count(*) FROM eddy_conversation_records) AS bodies,
(SELECT count(*) FROM eddy_conversation_events) AS links,
(SELECT rows FROM eddy_ingestion_budget WHERE key='conversation:global') AS charged`);
expect(r.rows[0]).toMatchObject({bodies:'1',links:'2',charged:'3'});
const e=await tx.execute(sql`SELECT e.task_id,r.event FROM eddy_conversation_events e JOIN eddy_conversation_records r ON r.id=e.record_id ORDER BY e.task_id`);
expect(e.rows.map((r:any)=>r.task_id)).toEqual(['a','b']); expect((e.rows[0] as any).event).toBe((e.rows[1] as any).event);
await tx.execute(sql`DROP SCHEMA eddy_migration_fixture CASCADE`);
});
});
run('new and old protocols share canonical bodies; duplicate deliveries charge nothing; owner isolation holds', async () => {
const owner = email('limit-dedupe'), v = batch();
const multi = {...v, tasks: [{taskId:'limit-a',taskTitle:'Fixture'},{taskId:'limit-b',taskTitle:'Second'}]};
const reply = await putConversation(owner, multi); expect(reply.taskIds).toEqual(['limit-a','limit-b']);
const q = await budget('conversation:owner:' + ownerKey(owner));
await putConversation(owner, multi); await putConversation(owner, {...v, taskId:'limit-b', taskTitle:'Second'});
expect(Number((await budget('conversation:owner:' + ownerKey(owner))).bytes)).toBe(Number(q.bytes));
expect(Number((await db.execute(sql`SELECT count(*) AS n FROM eddy_conversation_records WHERE owner = ${owner}`)).rows[0]!.n)).toBe(1);
for (const task of ['limit-a','limit-b']) expect((await getConversations(owner, conversationQuery({task}))).events.length).toBe(1);
expect((await getConversations(email('different'),conversationQuery({}))).events).toEqual([]);
await putConversation(email('different'), v);
expect(Number((await db.execute(sql`SELECT count(*) AS n FROM eddy_conversation_records WHERE owner = ${email('different')}`)).rows[0]!.n)).toBe(1);
});
run('daily and cumulative quotas atomically reject entire batches; exact retries still acknowledge', async () => {
const owner = email('limit-quota'); process.env.EDDY_CONVERSATION_DAILY_ROWS = '2';
await putConversation(owner, batch('first'));
const before = await totals();
await expect(putConversation(owner, batch('second'))).rejects.toMatchObject({code:'eddy_daily_quota'});
expect(await totals()).toEqual(before); await putConversation(owner, batch('first'));
delete process.env.EDDY_CONVERSATION_DAILY_ROWS;
process.env.EDDY_CONVERSATION_OWNER_BYTES = '1';
await expect(putConversation(owner, batch('third'))).rejects.toMatchObject({code:'eddy_storage_quota'});
expect(await totals()).toEqual(before); delete process.env.EDDY_CONVERSATION_OWNER_BYTES;
});
run('UTF-8 byte budget rejects a fresh batch before any payload write', async () => {
const before=await totals(); process.env.EDDY_CONVERSATION_DAILY_BYTES='1';
try { await expect(putConversation(email('limit-byte'),batch('unicode-byte'))).rejects.toMatchObject({code:'eddy_daily_quota'}); }
finally {delete process.env.EDDY_CONVERSATION_DAILY_BYTES;}
expect(await totals()).toEqual(before);
});
run('parallel owners cannot overshoot a shared global row cap', async () => {
const before = await totals(); process.env.EDDY_CONVERSATION_GLOBAL_ROWS = String(Number(before.rows) + 2);
const owners = Array.from({length:8},(_,i)=>`limit-race-${i}-${runTag}@example.com`);
const outcomes = await Promise.all(owners.map(o => putConversation(o,batch('race')).then(()=>true,e=> {expect(e).toBeInstanceOf(IngestionDenied);return false;})));
expect(outcomes.filter(Boolean).length).toBe(1);
for (let i=0;i<owners.length;i++) if (!outcomes[i]) await expect(putConversation(owners[i]!,batch('race'))).rejects.toMatchObject({code:'eddy_storage_quota'});
expect(Number((await totals()).rows)).toBe(Number(before.rows)+2);
const q=await budget('conversation:global'), actual=await totals(); expect(Number(q.rows)).toBe(Number(actual.rows)); expect(Number(q.bytes)).toBe(Number(actual.bytes));
delete process.env.EDDY_CONVERSATION_GLOBAL_ROWS;
});
run('per-owner and global rate buckets survive distinct callers; unconfigured ingestion fails closed', async () => {
process.env.EDDY_CONVERSATION_OWNER_BURST = '1'; process.env.EDDY_CONVERSATION_OWNER_RPM='1';
await admit('conversation',email('limit-rate')); await expect(admit('conversation',email('limit-rate'))).rejects.toMatchObject({code:'eddy_rate_limited'});
delete process.env.EDDY_CONVERSATION_OWNER_RPM; process.env.EDDY_CONVERSATION_OWNER_BURST='10000';
const key='conversation:global'; await db.execute(sql`UPDATE eddy_ingestion_budget SET tokens = 0, token_at = ${Date.now()} WHERE key = ${key}`);
await expect(admit('conversation',email('limit-other-rate'))).rejects.toMatchObject({code:'eddy_rate_limited'});
await db.execute(sql`UPDATE eddy_ingestion_budget SET tokens = NULL WHERE key = ${key}`);
const saved=process.env.EDDY_CONVERSATION_GLOBAL_BYTES; delete process.env.EDDY_CONVERSATION_GLOBAL_BYTES;
expect(()=>policy('conversation')).toThrow(IngestionDenied); process.env.EDDY_CONVERSATION_GLOBAL_BYTES=saved;
});
run('bounded retention removes links and orphan bodies, releases stored quota but preserves daily consumption', async () => {
const owner=email('limit-expired'); await putConversation(owner,batch('expired'));
const key='conversation:owner:'+ownerKey(owner), before=await budget(key);
await db.execute(sql`UPDATE eddy_conversation_events SET received_at = 1 WHERE owner = ${owner}`);
await db.execute(sql`UPDATE eddy_conversation_records SET received_at = 1 WHERE owner = ${owner}`);
await pruneConversations(); const after=await budget(key);
expect(Number(after.rows)).toBe(0); expect(Number(after.bytes)).toBe(0); expect(after.day_bytes).toBe(before.day_bytes);
const actual=await totals(), q=await budget('conversation:global'); expect(Number(q.bytes)).toBe(Number(actual.bytes)); expect(Number(q.rows)).toBe(Number(actual.rows));
});
afterAll(closePool);