Files
streamline/control-server/test/eddy_locks.integration.test.ts
2026-11-16 17:26:44 -08:00

122 lines
8.1 KiB
TypeScript

import './setup_env';
process.env.EDDY_RELEASE_TOKEN = 'release-token-for-lock-tests';
import { afterAll, expect, mock, test } from 'bun:test';
mock.module('../src/cluster', () => ({allEnrichedNodes: async () => [], callNodeAbsolute: async () => {throw new Error('No runtime execution');}}));
mock.module('../src/sso', () => ({resolveSsoEmail: async (h: Headers) => h.get('x-tt-session-v2') === 'lock-admin' ? 'lock-admin@example.com' : h.get('x-tt-session-v2') === 'lock-user' ? 'lock-user@example.com' : null}));
mock.module('../src/admin_db', () => ({isAdminPrefix: async (value: string) => value === 'lock-admin', normPrefix: (value: string) => value.split('@')[0]}));
const run = process.env.TEST_DATABASE_URL ? test : test.skip;
const { eddyApi } = await import('../src/routes_eddy');
const { db, closePool } = await import('../src/db_client');
const { sql } = await import('drizzle-orm');
const name = 'eddy-autorelease-test';
const pipeline = {authorization: 'Bearer release-token-for-lock-tests', 'x-eddy-user': 'autorelease'};
const admin = {'x-tt-session-v2': 'lock-admin'}, user = {'x-tt-session-v2': 'lock-user'};
const post = (action: string, body: unknown, headers: Record<string, string> = pipeline, lock = name) =>
eddyApi.request(`/admin/locks/${lock}/${action}`, {method: 'POST', headers: {'Content-Type': 'application/json', ...headers}, body: JSON.stringify(body)});
const json = async (response: Response) => ({status: response.status, body: await response.json() as any});
const clean = () => db.execute(sql`DELETE FROM eddy_locks WHERE name LIKE 'eddy-autorelease-test%'`);
run('only administrators or the release pipeline may use locks', async () => {
await clean();
expect((await post('register', {holder: 'a'}, {})).status).toBe(401);
expect((await post('register', {holder: 'a'}, user)).status).toBe(403);
expect((await post('register', {holder: 'a'}, {authorization: 'Bearer wrong'})).status).toBe(401);
expect((await eddyApi.request(`/admin/locks/${name}`, {headers: user})).status).toBe(403);
const r = await json(await post('register', {holder: 'admin-browser'}, admin));
expect(r.status).toBe(200);
expect(r.body.lock.actor).toBe('lock-admin');
expect((await post('unregister', {token: r.body.token}, admin)).status).toBe(200);
});
run('register, heartbeat and unregister with the token; others get 409 with the holder', async () => {
await clean();
const first = await json(await post('register', {holder: 'mac-a pid 1', ttlSeconds: 60, info: {host: 'mac-a', products: ['devicesim']}}));
expect(first.status).toBe(200);
expect(first.body.token).toMatch(/^[A-Za-z0-9_-]{43}$/);
expect(first.body.lock).toMatchObject({name, held: true, expired: false, holder: 'mac-a pid 1', actor: 'autorelease', info: {host: 'mac-a'}});
expect(first.body.lock.expiresAt - first.body.lock.acquiredAt).toBe(60_000);
const token = first.body.token;
const busy = await json(await post('register', {holder: 'mac-b pid 2'}));
expect(busy.status).toBe(409);
expect(busy.body.code).toBe('eddy_lock_held');
expect(busy.body.lock.holder).toBe('mac-a pid 1');
expect(JSON.stringify(busy.body)).not.toContain(token);
expect((await json(await post('heartbeat', {token: 'x'.repeat(43)}))).body.code).toBe('eddy_lock_lost');
await Bun.sleep(5);
const beat = await json(await post('heartbeat', {token, ttlSeconds: 120, status: {step: 'devicesim.package'}}));
expect(beat.status).toBe(200);
expect(beat.body.lock.heartbeatAt).toBeGreaterThan(first.body.lock.heartbeatAt);
expect(beat.body.lock.expiresAt - beat.body.lock.heartbeatAt).toBe(120_000);
expect(beat.body.lock.status).toEqual({step: 'devicesim.package'});
// A heartbeat without status keeps the last one.
expect((await json(await post('heartbeat', {token}))).body.lock.status).toEqual({step: 'devicesim.package'});
const view = await json(await eddyApi.request(`/admin/locks/${name}`, {headers: admin}));
expect(view.body).toMatchObject({held: true, holder: 'mac-a pid 1', status: {step: 'devicesim.package'}});
expect(JSON.stringify(view.body)).not.toContain(token);
expect((await post('unregister', {token: 'y'.repeat(43)})).status).toBe(409);
const done = await json(await post('unregister', {token, result: {published: ['devicesim 2026.11.12.1']}}));
expect(done.status).toBe(200);
expect(done.body.lock).toMatchObject({held: false, holder: null, last: {holder: 'mac-a pid 1', end: 'released', status: {step: 'devicesim.package'}, result: {published: ['devicesim 2026.11.12.1']}}});
// Released: the old token is dead, the name is free again.
expect((await post('heartbeat', {token})).status).toBe(409);
expect((await post('unregister', {token})).status).toBe(409);
const again = await json(await post('register', {holder: 'mac-b pid 2'}));
expect(again.status).toBe(200);
expect(again.body.lock.last.end).toBe('released');
await post('unregister', {token: again.body.token});
});
run('an expired lease can be renewed by its holder until somebody takes it over', async () => {
await clean();
const a = await json(await post('register', {holder: 'mac-a', ttlSeconds: 30}));
await db.execute(sql`UPDATE eddy_locks SET expires_at = ${Date.now() - 1000} WHERE name = ${name}`);
expect((await json(await eddyApi.request(`/admin/locks/${name}`, {headers: pipeline}))).body.expired).toBe(true);
// Nobody took it: the holder renews.
expect((await post('heartbeat', {token: a.body.token, ttlSeconds: 30})).status).toBe(200);
expect((await post('register', {holder: 'mac-b'})).status).toBe(409);
await db.execute(sql`UPDATE eddy_locks SET expires_at = ${Date.now() - 1000}, status = ${JSON.stringify({step: 'tiktokm.build'})}::jsonb WHERE name = ${name}`);
const b = await json(await post('register', {holder: 'mac-b', ttlSeconds: 30}));
expect(b.status).toBe(200);
expect(b.body.lock.holder).toBe('mac-b');
expect(b.body.lock.status).toBeNull();
expect(b.body.lock.last).toMatchObject({holder: 'mac-a', end: 'expired', status: {step: 'tiktokm.build'}});
// The old holder lost it: heartbeat and unregister both answer 409 and name the new holder.
const lost = await json(await post('heartbeat', {token: a.body.token}));
expect(lost.status).toBe(409);
expect(lost.body).toMatchObject({code: 'eddy_lock_lost', lock: {holder: 'mac-b'}});
expect((await post('unregister', {token: a.body.token})).status).toBe(409);
expect((await post('unregister', {token: b.body.token})).status).toBe(200);
});
run('concurrent registrations have exactly one winner; names are independent', async () => {
await clean();
const results = await Promise.all(Array.from({length: 12}, (_, i) => post('register', {holder: `racer ${i}`}).then(json)));
const winners = results.filter((r) => r.status === 200);
expect(winners).toHaveLength(1);
expect(results.filter((r) => r.status === 409).every((r) => r.body.lock.holder === winners[0].body.lock.holder)).toBe(true);
const other = await json(await post('register', {holder: 'other job'}, pipeline, name + '-other'));
expect(other.status).toBe(200);
const list = await json(await eddyApi.request('/admin/locks', {headers: admin}));
expect(list.body.locks.filter((l: any) => l.name.startsWith(name)).map((l: any) => l.name)).toEqual([name, name + '-other']);
await post('unregister', {token: winners[0].body.token});
await post('unregister', {token: other.body.token}, pipeline, name + '-other');
});
run('bad requests are rejected before touching the table', async () => {
for (const [action, body] of [
['register', {}], ['register', {holder: ''}], ['register', {holder: 'x'.repeat(129)}], ['register', {holder: 'a', ttlSeconds: 5}],
['register', {holder: 'a', ttlSeconds: 3601}], ['register', {holder: 'a', info: [1]}], ['register', {holder: 'a', info: {x: 'y'.repeat(5000)}}],
['heartbeat', {}], ['heartbeat', {token: 'short'}], ['unregister', {token: 'z'.repeat(43), result: 'done'}],
] as const) expect((await post(action, body)).status).toBe(400);
expect((await post('register', {holder: 'a'}, pipeline, 'Bad Name')).status).toBe(400);
expect((await eddyApi.request('/admin/locks/..%2Fx', {headers: admin})).status).toBe(400);
});
afterAll(async () => { if (process.env.TEST_DATABASE_URL) { await clean(); await closePool(); } });