import "./setup_env"; import { test, expect } from "bun:test"; import { createHash } from "node:crypto"; import type { EddyExportResult } from "@bbbug/shared/eddy_export"; // Run explicitly against a disposable database. Never derive this from DATABASE_URL. const url = process.env.TEST_DATABASE_URL; const local = url && new URL(url); if (local && (!["127.0.0.1", "localhost"].includes(local.hostname) || !local.pathname.endsWith("_test"))) throw new Error("Eddy export integration requires a disposable local *_test database"); test.skipIf(!url)("HTTP export lifecycle: authentication, concurrency, pinned commit, lease fencing, verification and expiry", async () => { const { db } = await import("../src/db_client"); const { features, nodes, eddy_feature_exports: jobs, eddy_export_notifications: notices } = await import("../src/schema"); const { eq } = await import("drizzle-orm"); const dal = await import("../src/eddy_export_db"); const { CLUSTER_SECRET_HEADER, TOS_DOWNLOAD_BASE } = await import("@bbbug/shared/protocol"); const env = { ...process.env }, originalFetch = globalThis.fetch; const id = "export-test-" + crypto.randomUUID(), nodeId = "export-node-" + crypto.randomUUID(); const hash = (b: Uint8Array | string) => createHash("sha256").update(b).digest("hex"); const recipe = (await import("@bbbug/shared/eddy_export")).exportRecipe("tiktok", "tiktok-ios")!; const snapshot = { commit: "a".repeat(40), start: { commit: "a".repeat(40), branch: "feature/eddy-test", from: "branch" }, recipe, recipeHash: hash(JSON.stringify(recipe)) }; const bytes = new TextEncoder().encode("fixture zip bytes"); let corrupt = false, drift = false, readbacks = 0; process.env.EDDY_EXPORT_ENABLED = "1"; process.env.CLUSTER_SECRET = "fixture-secret"; process.env.BITS_TOKEN = "fixture-token"; process.env.TOS_ACCESS_KEY = "fixture-ak"; process.env.TOS_SECRET_KEY = "fixture-sk"; globalThis.fetch = (async (input: any, init?: RequestInit) => { const target = String(input); if (target.endsWith("/invoke/getUserInfo")) return Response.json({ email: "export-fixture@example.invalid" }); if (target.endsWith("/fix-report")) return new Response("Fixture implementation progress"); if (target.includes("/openapi/merge_request/info?mr_id=123456")) return Response.json({ code: 0, data: { project_id: 114467, from_branch: "feature/eddy-test", target_branch: "develop", state: "merged", } }); if (target.endsWith("/sat/eddy-export/resolve")) { expect(new Headers(init?.headers).get(CLUSTER_SECRET_HEADER)).toBe("fixture-secret"); if (drift) await db.update(features).set({ attempts: 2 }).where(eq(features.id, id)); return Response.json(snapshot); } if (target.startsWith(TOS_DOWNLOAD_BASE + "/")) { readbacks++; return new Response(corrupt ? new Uint8Array([1]) : bytes); } throw new Error("Unexpected network request in fixture: " + target); }) as typeof fetch; const { eddyExportsApi: api } = await import("../src/routes_eddy_exports"); const post = (path: string, body: unknown, headers: Record = {}) => api.request(path, { method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify(body), }); const internal = (path: string, body: unknown) => post("/internal/eddy-exports/" + path, body, { [CLUSTER_SECRET_HEADER]: "fixture-secret" }); const create = () => post("/features/" + id + "/eddy-exports", {}, { "x-tt-session-v2": id }); try { await db.insert(nodes).values({ id: nodeId, baseUrl: "http://satellite.invalid", platform: "ios", status: "online", lastSeenAt: Date.now(), createdAt: Date.now() }); await db.insert(features).values({ id, title: "Eddy fixture", platform: "ios", status: "completed", priorityKey: 1, createdAt: Date.now(), branchName: "feature/eddy-test", repoKey: "tiktok-ios", mrUrl: "https://mr.invalid/1", executorNodeId: nodeId, watchers: "[]" }); expect((await post("/internal/eddy-exports/claim", { nodeId })).status).toBe(401); expect((await post("/features/" + id + "/eddy-exports", {})).status).toBe(401); const created = await Promise.all(Array.from({ length: 6 }, create)); for (const reply of created) { if (reply.status !== 202) throw new Error(await reply.text()); expect(reply.status).toBe(202); } const receipts = await Promise.all(created.map(r => r.json() as any)); expect(new Set(receipts.map(r => r.export.id)).size).toBe(1); const exportId = receipts[0].export.id as string; const download = "/features/" + id + "/eddy-exports/" + exportId + "/app"; expect((await api.request(download)).status).toBe(410); expect((await dal.claimExport(nodeId + "-other"))).toBeNull(); const claims = await Promise.all([dal.claimExport(nodeId), dal.claimExport(nodeId)]); expect(claims.filter(Boolean)).toHaveLength(1); const job = claims.find(Boolean)!; expect(job.commit).toBe(snapshot.commit); expect(job.instructions).toContain("Fixture implementation progress"); expect(job.instructions).toContain(snapshot.commit); const owner = { nodeId, lease: job.lease }; expect((await internal(exportId + "/update", { ...owner, lease: "stale", phase: "failed" })).status).toBe(409); expect((await internal(exportId + "/update", { ...owner, phase: "ready" })).status).toBe(400); expect((await internal(exportId + "/update", { ...owner, phase: "uploading" })).status).toBe(409); expect(await dal.claimExportNotice()).toBeNull(); for (const phase of ["packaging", "verifying", "uploading"]) expect((await internal(exportId + "/update", { ...owner, phase })).status).toBe(200); const bid = "tiktokm-47-470000-12345678"; const result: EddyExportResult = { baselineId: bid, commit: job.commit, eddyVersion: "fixture", devicesimSha256: "b".repeat(64), artifacts: { app: { file: "baseline-" + bid + ".zip", bytes: bytes.length, sha256: hash(bytes) }, source: { file: "baseline-" + bid + "-source.zip", bytes: bytes.length, sha256: hash(bytes) }, } }; expect((await internal(exportId + "/upload", { ...owner, kind: "app", result: { ...result, commit: "c".repeat(40) } })).status).toBe(409); for (const kind of ["app", "source"]) { const upload = await internal(exportId + "/upload", { ...owner, kind, result }); expect(upload.status).toBe(200); const signed = await upload.json() as { url: string; headers: Record }; const target = new URL(signed.url); expect(target.host).toBe("streamline.tos-cn-north.byted.org"); expect(target.pathname).toBe("/eddy/task-exports/" + exportId + "/" + hash(bytes) + "/" + result.artifacts[kind as "app" | "source"].file); expect(target.searchParams.get("tos-signature")).toMatch(/^[A-Za-z0-9_-]{30}$/); expect(signed.url).not.toContain("fixture-ak"); expect(signed.headers["content-length"]).toBe(String(bytes.length)); } const changed = structuredClone(result); changed.artifacts.app.bytes++; expect((await internal(exportId + "/upload", { ...owner, kind: "app", result: changed })).status).toBe(409); expect(await dal.claimExportNotice()).toBeNull(); expect((await internal(exportId + "/complete", owner)).status).toBe(202); for (let n = 0; n < 100 && (await dal.getExport(exportId))?.phase === "uploading"; n++) await Bun.sleep(5); expect((await dal.getExport(exportId))?.phase).toBe("ready"); const checks = readbacks; expect((await internal(exportId + "/complete", owner)).status).toBe(200); expect(readbacks).toBe(checks); // lost final response is idempotent expect((await api.request(download)).status).toBe(302); expect((await api.request(download)).headers.get("location")).toContain(hash(bytes)); expect(await (await api.request(download.replace(/\/app$/, "/relay"))).text()).toBe(job.instructions); const notice = await dal.claimExportNotice(); expect(notice?.email).toBe("export-fixture@example.invalid"); expect(await dal.claimExportNotice()).toBeNull(); // reserve, don't duplicate on next tick await dal.sentExportNotice(exportId, notice!.email); const reuse = await create(); expect((await reuse.json() as any).export.id).toBe(exportId); expect(await dal.claimExportNotice()).toBeNull(); expect((await db.select().from(notices).where(eq(notices.exportId, exportId)))).toHaveLength(1); expect((await db.select().from(features).where(eq(features.id, id)))[0]!.status).toBe("completed"); await db.update(jobs).set({ expiresAt: Date.now() - 1 }).where(eq(jobs.id, exportId)); expect((await api.request(download)).status).toBe(410); await dal.sweepExports(); const retry = (await (await create()).json() as any).export; expect(retry.id).not.toBe(exportId); const bad = await dal.claimExport(nodeId); const badOwner = { nodeId, lease: bad!.lease }; for (const phase of ["packaging", "verifying", "uploading"] as const) await dal.updateExport(bad!.id, nodeId, bad!.lease, phase); expect((await internal(bad!.id + "/upload", { ...badOwner, kind: "app", result })).status).toBe(200); corrupt = true; expect((await internal(bad!.id + "/complete", badOwner)).status).toBe(202); for (let n = 0; n < 100 && (await dal.getExport(bad!.id))?.phase === "uploading"; n++) await Bun.sleep(5); expect((await dal.getExport(bad!.id))?.phase).toBe("failed"); expect((await internal(bad!.id + "/complete", badOwner)).status).toBe(409); expect(await dal.claimExportNotice()).toBeNull(); corrupt = false; await create(); const next = await dal.claimExport(nodeId); await db.update(jobs).set({ leaseUntil: Date.now() - 1 }).where(eq(jobs.id, next!.id)); await expect(dal.updateExport(next!.id, nodeId, next!.lease, "packaging")).rejects.toThrow("租约"); await dal.sweepExports(); expect((await dal.getExport(next!.id))?.phase).toBe("failed"); await expect(dal.updateExport(next!.id, nodeId, next!.lease, "building")).rejects.toThrow("租约"); drift = true; expect((await create()).status).toBe(409); expect(await dal.claimExport(nodeId)).toBeNull(); drift = false; await db.update(features).set({ mrUrl: "https://bits.bytedance.net/devops/1500037378/code/detail/123456" }).where(eq(features.id, id)); snapshot.start = { ...snapshot.start, from: "base", branch: "develop" }; const mergedReply = await create(); if (mergedReply.status !== 202) throw new Error(await mergedReply.text()); const mergedExport = (await mergedReply.json() as any).export; const mergedJob = await dal.claimExport(nodeId); expect(mergedJob!.branch).toBe("develop"); expect(mergedJob!.instructions).toContain("已经合入"); expect(mergedJob!.instructions).toContain("-followup origin/develop"); await db.update(features).set({ remark: "New feedback for the local relay" }).where(eq(features.id, id)); const updatedContext = (await (await create()).json() as any).export; expect(updatedContext.id).not.toBe(mergedExport.id); await db.update(features).set({ platform: "android" }).where(eq(features.id, id)); expect((await create()).status).toBe(409); const android = await (await api.request("/features/" + id + "/eddy-exports")).json() as any; expect(android).toEqual({ enabled: false, exports: [] }); } finally { globalThis.fetch = originalFetch; for (const key of Object.keys(process.env)) if (!(key in env)) delete process.env[key]; Object.assign(process.env, env); await db.delete(features).where(eq(features.id, id)); await db.delete(nodes).where(eq(nodes.id, nodeId)); } }, 30_000); test.skipIf(!url)("an offline executor hands the export to a live peer, and a queued job stranded on a gone node is released", async () => { const { db } = await import("../src/db_client"); const { features, nodes, node_clusters, eddy_feature_exports: jobs } = await import("../src/schema"); const { eq, inArray } = await import("drizzle-orm"); const dal = await import("../src/eddy_export_db"); const env = { ...process.env }, originalFetch = globalThis.fetch; const tag = crypto.randomUUID(), id = "export-fallback-" + tag; const ids = { home: "export-home-" + tag, recent: "export-recent-" + tag, failing: "export-a-" + tag, peer: "export-b-" + tag, lab: "export-lab-" + tag, lemon: "export-lemon-" + tag }; const hash = (b: string) => createHash("sha256").update(b).digest("hex"); const recipe = (await import("@bbbug/shared/eddy_export")).exportRecipe("tiktok", "tiktok-ios")!; const snapshot = { commit: "d".repeat(40), start: { commit: "d".repeat(40), branch: "feature/eddy-fallback", from: "branch" }, recipe, recipeHash: hash(JSON.stringify(recipe)) }; const asked: string[] = []; process.env.EDDY_EXPORT_ENABLED = "1"; process.env.TOS_ACCESS_KEY = "fixture-ak"; process.env.TOS_SECRET_KEY = "fixture-sk"; globalThis.fetch = (async (input: any) => { const target = String(input); if (target.endsWith("/invoke/getUserInfo")) return Response.json({ email: "export-fixture@example.invalid" }); if (target.endsWith("/sat/eddy-export/resolve")) { asked.push(new URL(target).hostname); if (target.startsWith("http://" + ids.peer + ".invalid")) return Response.json(snapshot); return Response.json({ error: "卫星机没有 airb,无法远程构建" }, { status: 409 }); } throw new Error("Unexpected network request in fixture: " + target); }) as typeof fetch; const { eddyExportsApi: api } = await import("../src/routes_eddy_exports"); const create = () => api.request("/features/" + id + "/eddy-exports", { method: "POST", headers: { "x-tt-session-v2": id } }); const now = Date.now(); const node = (nodeId: string, status: string, lastSeenAt: number) => ({ id: nodeId, baseUrl: "http://" + nodeId + ".invalid", platform: "ios", status, lastSeenAt, createdAt: now }); try { await db.insert(nodes).values([ node(ids.home, "offline", now - 20 * 60_000), node(ids.recent, "offline", now - 60_000), node(ids.failing, "online", now), node(ids.peer, "online", now), node(ids.lab, "online", now), node(ids.lemon, "online", now), ]); await db.insert(node_clusters).values([ { nodeId: ids.home, cluster: "default" }, { nodeId: ids.recent, cluster: "default" }, { nodeId: ids.failing, cluster: "default" }, { nodeId: ids.peer, cluster: "default" }, { nodeId: ids.lab, cluster: "manual-only" }, { nodeId: ids.lab, cluster: "default" }, { nodeId: ids.lemon, cluster: "lemon-" + tag }, ]); await db.insert(features).values({ id, title: "Eddy fallback fixture", platform: "ios", status: "manual_review", priorityKey: 1, createdAt: now, branchName: "feature/eddy-fallback", repoKey: "tiktok-ios", mrUrl: "https://mr.invalid/2", executorNodeId: ids.home, watchers: "[]" }); const reply = await create(); if (reply.status !== 202) throw new Error(await reply.text()); const exportId = (await reply.json() as any).export.id as string; expect((await dal.getExport(exportId))?.nodeId).toBe(ids.peer); expect(asked).toContain(ids.failing + ".invalid"); for (const skipped of [ids.home, ids.lab, ids.lemon]) expect(asked).not.toContain(skipped + ".invalid"); expect(await dal.claimExport(ids.home)).toBeNull(); const job = await dal.claimExport(ids.peer); expect(job?.id).toBe(exportId); expect(job?.nodeId).toBe(ids.peer); expect(job?.instructions).toContain("未能从执行节点获取报告"); // A running export occupies the node's fix slot and is listed for the node strip. const { countSlotsPerNode } = await import("../src/db"); expect((await countSlotsPerNode()).get(ids.peer)?.fixInFlight).toBe(1); const running = await (await api.request("/eddy-exports/running")).json() as any; expect(running.exports).toContainEqual({ id: exportId, featureId: id, title: "Eddy fallback fixture", nodeId: ids.peer, phase: "building" }); const listed = await (await api.request("/features/" + id + "/eddy-exports")).json() as any; expect(listed.exports[0].nodeId).toBe(ids.peer); expect((await db.select().from(features).where(eq(features.id, id)))[0]!.executorNodeId).toBe(ids.home); await db.update(nodes).set({ status: "offline", lastSeenAt: now - 60_000 }).where(eq(nodes.id, ids.peer)); const stuck = await create(); expect(stuck.status).toBe(409); expect(await stuck.text()).toContain("任务卫星机离线"); const queue = async (nodeId: string) => (await dal.enqueueExport({ ...snapshot, featureId: id, title: "fixture", branch: snapshot.start.branch, mrUrl: "https://mr.invalid/2", nodeId, contextHash: nodeId, instructions: "fixture" } as any, "export-fixture@example.invalid")).id; const gone = await queue(ids.home), blip = await queue(ids.recent), alive = await queue(ids.failing); await dal.sweepExports(); expect((await dal.getExport(gone))?.phase).toBe("failed"); expect((await dal.getExport(gone))?.error).toContain("改派"); expect((await dal.getExport(blip))?.phase).toBe("queued"); expect((await dal.getExport(alive))?.phase).toBe("queued"); } finally { globalThis.fetch = originalFetch; for (const key of Object.keys(process.env)) if (!(key in env)) delete process.env[key]; Object.assign(process.env, env); await db.delete(features).where(eq(features.id, id)); await db.delete(node_clusters).where(inArray(node_clusters.nodeId, Object.values(ids))); await db.delete(nodes).where(inArray(nodes.id, Object.values(ids))); } }, 30_000);