256 lines
17 KiB
TypeScript
256 lines
17 KiB
TypeScript
import "./setup_env";
|
|
import { test, expect } from "bun:test";
|
|
import { createHash } from "node:crypto";
|
|
import type { EddyExportResult } from "@bbbug/shared/eddy_export";
|
|
|
|
// Run explicitly against a disposable database. Never derive this from DATABASE_URL.
|
|
const url = process.env.TEST_DATABASE_URL;
|
|
const local = url && new URL(url);
|
|
if (local && (!["127.0.0.1", "localhost"].includes(local.hostname) || !local.pathname.endsWith("_test")))
|
|
throw new Error("Eddy export integration requires a disposable local *_test database");
|
|
|
|
test.skipIf(!url)("HTTP export lifecycle: authentication, concurrency, pinned commit, lease fencing, verification and expiry", async () => {
|
|
const { db } = await import("../src/db_client");
|
|
const { features, nodes, eddy_feature_exports: jobs, eddy_export_notifications: notices } = await import("../src/schema");
|
|
const { eq } = await import("drizzle-orm");
|
|
const dal = await import("../src/eddy_export_db");
|
|
const { CLUSTER_SECRET_HEADER, TOS_DOWNLOAD_BASE } = await import("@bbbug/shared/protocol");
|
|
const env = { ...process.env }, originalFetch = globalThis.fetch;
|
|
const id = "export-test-" + crypto.randomUUID(), nodeId = "export-node-" + crypto.randomUUID();
|
|
const hash = (b: Uint8Array | string) => createHash("sha256").update(b).digest("hex");
|
|
const recipe = (await import("@bbbug/shared/eddy_export")).exportRecipe("tiktok", "tiktok-ios")!;
|
|
const snapshot = { commit: "a".repeat(40), start: { commit: "a".repeat(40), branch: "feature/eddy-test", from: "branch" }, recipe, recipeHash: hash(JSON.stringify(recipe)) };
|
|
const bytes = new TextEncoder().encode("fixture zip bytes");
|
|
let corrupt = false, drift = false, readbacks = 0;
|
|
process.env.EDDY_EXPORT_ENABLED = "1";
|
|
process.env.CLUSTER_SECRET = "fixture-secret";
|
|
process.env.BITS_TOKEN = "fixture-token";
|
|
process.env.TOS_ACCESS_KEY = "fixture-ak"; process.env.TOS_SECRET_KEY = "fixture-sk";
|
|
globalThis.fetch = (async (input: any, init?: RequestInit) => {
|
|
const target = String(input);
|
|
if (target.endsWith("/invoke/getUserInfo")) return Response.json({ email: "export-fixture@example.invalid" });
|
|
if (target.endsWith("/fix-report")) return new Response("Fixture implementation progress");
|
|
if (target.includes("/openapi/merge_request/info?mr_id=123456")) return Response.json({ code: 0, data: {
|
|
project_id: 114467, from_branch: "feature/eddy-test", target_branch: "develop", state: "merged",
|
|
} });
|
|
if (target.endsWith("/sat/eddy-export/resolve")) {
|
|
expect(new Headers(init?.headers).get(CLUSTER_SECRET_HEADER)).toBe("fixture-secret");
|
|
if (drift) await db.update(features).set({ attempts: 2 }).where(eq(features.id, id));
|
|
return Response.json(snapshot);
|
|
}
|
|
if (target.startsWith(TOS_DOWNLOAD_BASE + "/")) {
|
|
readbacks++; return new Response(corrupt ? new Uint8Array([1]) : bytes);
|
|
}
|
|
throw new Error("Unexpected network request in fixture: " + target);
|
|
}) as typeof fetch;
|
|
const { eddyExportsApi: api } = await import("../src/routes_eddy_exports");
|
|
const post = (path: string, body: unknown, headers: Record<string, string> = {}) => api.request(path, {
|
|
method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify(body),
|
|
});
|
|
const internal = (path: string, body: unknown) => post("/internal/eddy-exports/" + path, body, { [CLUSTER_SECRET_HEADER]: "fixture-secret" });
|
|
const create = () => post("/features/" + id + "/eddy-exports", {}, { "x-tt-session-v2": id });
|
|
try {
|
|
await db.insert(nodes).values({ id: nodeId, baseUrl: "http://satellite.invalid", platform: "ios", status: "online", lastSeenAt: Date.now(), createdAt: Date.now() });
|
|
await db.insert(features).values({ id, title: "Eddy fixture", platform: "ios", status: "completed", priorityKey: 1, createdAt: Date.now(),
|
|
branchName: "feature/eddy-test", repoKey: "tiktok-ios", mrUrl: "https://mr.invalid/1", executorNodeId: nodeId, watchers: "[]" });
|
|
expect((await post("/internal/eddy-exports/claim", { nodeId })).status).toBe(401);
|
|
expect((await post("/features/" + id + "/eddy-exports", {})).status).toBe(401);
|
|
const created = await Promise.all(Array.from({ length: 6 }, create));
|
|
for (const reply of created) { if (reply.status !== 202) throw new Error(await reply.text()); expect(reply.status).toBe(202); }
|
|
const receipts = await Promise.all(created.map(r => r.json() as any));
|
|
expect(new Set(receipts.map(r => r.export.id)).size).toBe(1);
|
|
const exportId = receipts[0].export.id as string;
|
|
const download = "/features/" + id + "/eddy-exports/" + exportId + "/app";
|
|
expect((await api.request(download)).status).toBe(410);
|
|
expect((await dal.claimExport(nodeId + "-other"))).toBeNull();
|
|
const claims = await Promise.all([dal.claimExport(nodeId), dal.claimExport(nodeId)]);
|
|
expect(claims.filter(Boolean)).toHaveLength(1);
|
|
const job = claims.find(Boolean)!;
|
|
expect(job.commit).toBe(snapshot.commit);
|
|
expect(job.instructions).toContain("Fixture implementation progress");
|
|
expect(job.instructions).toContain(snapshot.commit);
|
|
const owner = { nodeId, lease: job.lease };
|
|
expect((await internal(exportId + "/update", { ...owner, lease: "stale", phase: "failed" })).status).toBe(409);
|
|
expect((await internal(exportId + "/update", { ...owner, phase: "ready" })).status).toBe(400);
|
|
expect((await internal(exportId + "/update", { ...owner, phase: "uploading" })).status).toBe(409);
|
|
expect(await dal.claimExportNotice()).toBeNull();
|
|
for (const phase of ["packaging", "verifying", "uploading"])
|
|
expect((await internal(exportId + "/update", { ...owner, phase })).status).toBe(200);
|
|
const bid = "tiktokm-47-470000-12345678";
|
|
const result: EddyExportResult = { baselineId: bid, commit: job.commit, eddyVersion: "fixture", devicesimSha256: "b".repeat(64),
|
|
artifacts: {
|
|
app: { file: "baseline-" + bid + ".zip", bytes: bytes.length, sha256: hash(bytes) },
|
|
source: { file: "baseline-" + bid + "-source.zip", bytes: bytes.length, sha256: hash(bytes) },
|
|
} };
|
|
expect((await internal(exportId + "/upload", { ...owner, kind: "app", result: { ...result, commit: "c".repeat(40) } })).status).toBe(409);
|
|
for (const kind of ["app", "source"]) {
|
|
const upload = await internal(exportId + "/upload", { ...owner, kind, result });
|
|
expect(upload.status).toBe(200);
|
|
const signed = await upload.json() as { url: string; headers: Record<string, string> };
|
|
const target = new URL(signed.url);
|
|
expect(target.host).toBe("streamline.tos-cn-north.byted.org");
|
|
expect(target.pathname).toBe("/eddy/task-exports/" + exportId + "/" + hash(bytes) + "/" + result.artifacts[kind as "app" | "source"].file);
|
|
expect(target.searchParams.get("tos-signature")).toMatch(/^[A-Za-z0-9_-]{30}$/);
|
|
expect(signed.url).not.toContain("fixture-ak");
|
|
expect(signed.headers["content-length"]).toBe(String(bytes.length));
|
|
}
|
|
const changed = structuredClone(result); changed.artifacts.app.bytes++;
|
|
expect((await internal(exportId + "/upload", { ...owner, kind: "app", result: changed })).status).toBe(409);
|
|
expect(await dal.claimExportNotice()).toBeNull();
|
|
expect((await internal(exportId + "/complete", owner)).status).toBe(202);
|
|
for (let n = 0; n < 100 && (await dal.getExport(exportId))?.phase === "uploading"; n++) await Bun.sleep(5);
|
|
expect((await dal.getExport(exportId))?.phase).toBe("ready");
|
|
const checks = readbacks;
|
|
expect((await internal(exportId + "/complete", owner)).status).toBe(200);
|
|
expect(readbacks).toBe(checks); // lost final response is idempotent
|
|
expect((await api.request(download)).status).toBe(302);
|
|
expect((await api.request(download)).headers.get("location")).toContain(hash(bytes));
|
|
expect(await (await api.request(download.replace(/\/app$/, "/relay"))).text()).toBe(job.instructions);
|
|
const notice = await dal.claimExportNotice();
|
|
expect(notice?.email).toBe("export-fixture@example.invalid");
|
|
expect(await dal.claimExportNotice()).toBeNull(); // reserve, don't duplicate on next tick
|
|
await dal.sentExportNotice(exportId, notice!.email);
|
|
const reuse = await create();
|
|
expect((await reuse.json() as any).export.id).toBe(exportId);
|
|
expect(await dal.claimExportNotice()).toBeNull();
|
|
expect((await db.select().from(notices).where(eq(notices.exportId, exportId)))).toHaveLength(1);
|
|
expect((await db.select().from(features).where(eq(features.id, id)))[0]!.status).toBe("completed");
|
|
|
|
await db.update(jobs).set({ expiresAt: Date.now() - 1 }).where(eq(jobs.id, exportId));
|
|
expect((await api.request(download)).status).toBe(410);
|
|
await dal.sweepExports();
|
|
const retry = (await (await create()).json() as any).export;
|
|
expect(retry.id).not.toBe(exportId);
|
|
const bad = await dal.claimExport(nodeId);
|
|
const badOwner = { nodeId, lease: bad!.lease };
|
|
for (const phase of ["packaging", "verifying", "uploading"] as const) await dal.updateExport(bad!.id, nodeId, bad!.lease, phase);
|
|
expect((await internal(bad!.id + "/upload", { ...badOwner, kind: "app", result })).status).toBe(200);
|
|
corrupt = true;
|
|
expect((await internal(bad!.id + "/complete", badOwner)).status).toBe(202);
|
|
for (let n = 0; n < 100 && (await dal.getExport(bad!.id))?.phase === "uploading"; n++) await Bun.sleep(5);
|
|
expect((await dal.getExport(bad!.id))?.phase).toBe("failed");
|
|
expect((await internal(bad!.id + "/complete", badOwner)).status).toBe(409);
|
|
expect(await dal.claimExportNotice()).toBeNull();
|
|
corrupt = false;
|
|
await create();
|
|
const next = await dal.claimExport(nodeId);
|
|
await db.update(jobs).set({ leaseUntil: Date.now() - 1 }).where(eq(jobs.id, next!.id));
|
|
await expect(dal.updateExport(next!.id, nodeId, next!.lease, "packaging")).rejects.toThrow("租约");
|
|
await dal.sweepExports();
|
|
expect((await dal.getExport(next!.id))?.phase).toBe("failed");
|
|
await expect(dal.updateExport(next!.id, nodeId, next!.lease, "building")).rejects.toThrow("租约");
|
|
drift = true;
|
|
expect((await create()).status).toBe(409);
|
|
expect(await dal.claimExport(nodeId)).toBeNull();
|
|
drift = false;
|
|
await db.update(features).set({ mrUrl: "https://bits.bytedance.net/devops/1500037378/code/detail/123456" }).where(eq(features.id, id));
|
|
snapshot.start = { ...snapshot.start, from: "base", branch: "develop" };
|
|
const mergedReply = await create();
|
|
if (mergedReply.status !== 202) throw new Error(await mergedReply.text());
|
|
const mergedExport = (await mergedReply.json() as any).export;
|
|
const mergedJob = await dal.claimExport(nodeId);
|
|
expect(mergedJob!.branch).toBe("develop");
|
|
expect(mergedJob!.instructions).toContain("已经合入");
|
|
expect(mergedJob!.instructions).toContain("-followup origin/develop");
|
|
await db.update(features).set({ remark: "New feedback for the local relay" }).where(eq(features.id, id));
|
|
const updatedContext = (await (await create()).json() as any).export;
|
|
expect(updatedContext.id).not.toBe(mergedExport.id);
|
|
await db.update(features).set({ platform: "android" }).where(eq(features.id, id));
|
|
expect((await create()).status).toBe(409);
|
|
const android = await (await api.request("/features/" + id + "/eddy-exports")).json() as any;
|
|
expect(android).toEqual({ enabled: false, exports: [] });
|
|
} finally {
|
|
globalThis.fetch = originalFetch;
|
|
for (const key of Object.keys(process.env)) if (!(key in env)) delete process.env[key];
|
|
Object.assign(process.env, env);
|
|
await db.delete(features).where(eq(features.id, id));
|
|
await db.delete(nodes).where(eq(nodes.id, nodeId));
|
|
}
|
|
}, 30_000);
|
|
|
|
test.skipIf(!url)("an offline executor hands the export to a live peer, and a queued job stranded on a gone node is released", async () => {
|
|
const { db } = await import("../src/db_client");
|
|
const { features, nodes, node_clusters, eddy_feature_exports: jobs } = await import("../src/schema");
|
|
const { eq, inArray } = await import("drizzle-orm");
|
|
const dal = await import("../src/eddy_export_db");
|
|
const env = { ...process.env }, originalFetch = globalThis.fetch;
|
|
const tag = crypto.randomUUID(), id = "export-fallback-" + tag;
|
|
const ids = { home: "export-home-" + tag, recent: "export-recent-" + tag, failing: "export-a-" + tag, peer: "export-b-" + tag,
|
|
lab: "export-lab-" + tag, lemon: "export-lemon-" + tag };
|
|
const hash = (b: string) => createHash("sha256").update(b).digest("hex");
|
|
const recipe = (await import("@bbbug/shared/eddy_export")).exportRecipe("tiktok", "tiktok-ios")!;
|
|
const snapshot = { commit: "d".repeat(40), start: { commit: "d".repeat(40), branch: "feature/eddy-fallback", from: "branch" }, recipe, recipeHash: hash(JSON.stringify(recipe)) };
|
|
const asked: string[] = [];
|
|
process.env.EDDY_EXPORT_ENABLED = "1";
|
|
process.env.TOS_ACCESS_KEY = "fixture-ak"; process.env.TOS_SECRET_KEY = "fixture-sk";
|
|
globalThis.fetch = (async (input: any) => {
|
|
const target = String(input);
|
|
if (target.endsWith("/invoke/getUserInfo")) return Response.json({ email: "export-fixture@example.invalid" });
|
|
if (target.endsWith("/sat/eddy-export/resolve")) {
|
|
asked.push(new URL(target).hostname);
|
|
if (target.startsWith("http://" + ids.peer + ".invalid")) return Response.json(snapshot);
|
|
return Response.json({ error: "卫星机没有 airb,无法远程构建" }, { status: 409 });
|
|
}
|
|
throw new Error("Unexpected network request in fixture: " + target);
|
|
}) as typeof fetch;
|
|
const { eddyExportsApi: api } = await import("../src/routes_eddy_exports");
|
|
const create = () => api.request("/features/" + id + "/eddy-exports", { method: "POST", headers: { "x-tt-session-v2": id } });
|
|
const now = Date.now();
|
|
const node = (nodeId: string, status: string, lastSeenAt: number) =>
|
|
({ id: nodeId, baseUrl: "http://" + nodeId + ".invalid", platform: "ios", status, lastSeenAt, createdAt: now });
|
|
try {
|
|
await db.insert(nodes).values([
|
|
node(ids.home, "offline", now - 20 * 60_000), node(ids.recent, "offline", now - 60_000),
|
|
node(ids.failing, "online", now), node(ids.peer, "online", now), node(ids.lab, "online", now), node(ids.lemon, "online", now),
|
|
]);
|
|
await db.insert(node_clusters).values([
|
|
{ nodeId: ids.home, cluster: "default" }, { nodeId: ids.recent, cluster: "default" },
|
|
{ nodeId: ids.failing, cluster: "default" }, { nodeId: ids.peer, cluster: "default" },
|
|
{ nodeId: ids.lab, cluster: "manual-only" }, { nodeId: ids.lab, cluster: "default" }, { nodeId: ids.lemon, cluster: "lemon-" + tag },
|
|
]);
|
|
await db.insert(features).values({ id, title: "Eddy fallback fixture", platform: "ios", status: "manual_review", priorityKey: 1, createdAt: now,
|
|
branchName: "feature/eddy-fallback", repoKey: "tiktok-ios", mrUrl: "https://mr.invalid/2", executorNodeId: ids.home, watchers: "[]" });
|
|
|
|
const reply = await create();
|
|
if (reply.status !== 202) throw new Error(await reply.text());
|
|
const exportId = (await reply.json() as any).export.id as string;
|
|
expect((await dal.getExport(exportId))?.nodeId).toBe(ids.peer);
|
|
expect(asked).toContain(ids.failing + ".invalid");
|
|
for (const skipped of [ids.home, ids.lab, ids.lemon]) expect(asked).not.toContain(skipped + ".invalid");
|
|
expect(await dal.claimExport(ids.home)).toBeNull();
|
|
const job = await dal.claimExport(ids.peer);
|
|
expect(job?.id).toBe(exportId);
|
|
expect(job?.nodeId).toBe(ids.peer);
|
|
expect(job?.instructions).toContain("未能从执行节点获取报告");
|
|
// A running export occupies the node's fix slot and is listed for the node strip.
|
|
const { countSlotsPerNode } = await import("../src/db");
|
|
expect((await countSlotsPerNode()).get(ids.peer)?.fixInFlight).toBe(1);
|
|
const running = await (await api.request("/eddy-exports/running")).json() as any;
|
|
expect(running.exports).toContainEqual({ id: exportId, featureId: id, title: "Eddy fallback fixture", nodeId: ids.peer, phase: "building" });
|
|
const listed = await (await api.request("/features/" + id + "/eddy-exports")).json() as any;
|
|
expect(listed.exports[0].nodeId).toBe(ids.peer);
|
|
expect((await db.select().from(features).where(eq(features.id, id)))[0]!.executorNodeId).toBe(ids.home);
|
|
|
|
await db.update(nodes).set({ status: "offline", lastSeenAt: now - 60_000 }).where(eq(nodes.id, ids.peer));
|
|
const stuck = await create();
|
|
expect(stuck.status).toBe(409);
|
|
expect(await stuck.text()).toContain("任务卫星机离线");
|
|
|
|
const queue = async (nodeId: string) => (await dal.enqueueExport({ ...snapshot, featureId: id, title: "fixture", branch: snapshot.start.branch,
|
|
mrUrl: "https://mr.invalid/2", nodeId, contextHash: nodeId, instructions: "fixture" } as any, "export-fixture@example.invalid")).id;
|
|
const gone = await queue(ids.home), blip = await queue(ids.recent), alive = await queue(ids.failing);
|
|
await dal.sweepExports();
|
|
expect((await dal.getExport(gone))?.phase).toBe("failed");
|
|
expect((await dal.getExport(gone))?.error).toContain("改派");
|
|
expect((await dal.getExport(blip))?.phase).toBe("queued");
|
|
expect((await dal.getExport(alive))?.phase).toBe("queued");
|
|
} finally {
|
|
globalThis.fetch = originalFetch;
|
|
for (const key of Object.keys(process.env)) if (!(key in env)) delete process.env[key];
|
|
Object.assign(process.env, env);
|
|
await db.delete(features).where(eq(features.id, id));
|
|
await db.delete(node_clusters).where(inArray(node_clusters.nodeId, Object.values(ids)));
|
|
await db.delete(nodes).where(inArray(nodes.id, Object.values(ids)));
|
|
}
|
|
}, 30_000);
|