307 lines
16 KiB
TypeScript
307 lines
16 KiB
TypeScript
#!/usr/bin/env bun
|
||
/**
|
||
* 卫星热部署 CLI — the non-interactive twin of 管理 → 卫星热部署.
|
||
*
|
||
* Exists so a deploy can finish the job: `deploy.sh` builds the bundle, then
|
||
* calls this to actually get it onto the nodes, instead of ending with a note
|
||
* telling a human to go click three buttons (which is how nodes ended up
|
||
* running week-old code while the panel cheerfully said 「最新」).
|
||
*
|
||
* Why it is safe to automate, when the panel deliberately kept it manual:
|
||
* the panel's danger is 强制部署 — swapping the bundle exits the satellite
|
||
* process and kills whatever it was running. This script NEVER does that by
|
||
* default. A busy node gets 「空档部署」: the node persists the bundle, stops
|
||
* claiming new work, finishes its current task, and swaps in the gap. Nothing
|
||
* is interrupted, so there is no judgement call left for a human to make.
|
||
* `--force` restores the destructive behaviour and is opt-in only.
|
||
*
|
||
* Why it talks HTTP instead of importing the control plane's own route:
|
||
* - `POST /api/deploy` is SSO+admin gated (browser credentials) — a script
|
||
* has none, and weakening that gate to let scripts in would be a far worse
|
||
* trade than this file.
|
||
* - `pendingDeploy` / `supportsQueuedDeploy` live in the control-server's
|
||
* MEMORY (learned from heartbeats, see cluster.ts), not the DB, so a
|
||
* separate process must ask the running server for them — `GET /api/nodes`
|
||
* is read-only and unauthenticated.
|
||
* So: read node state from the control plane, push bytes straight to each node
|
||
* over the same cluster-secret channel the control plane itself uses.
|
||
*
|
||
* Run it ON the control-plane box (it needs CLUSTER_SECRET from .env, the built
|
||
* bundle, and network to the nodes):
|
||
*
|
||
* bun scripts/push-satellites.ts # push to the `default` pool
|
||
* bun scripts/push-satellites.ts --dry-run # print the plan, touch nothing
|
||
* bun scripts/push-satellites.ts --all-clusters # every pool, incl. other teams'
|
||
* bun scripts/push-satellites.ts --cluster pns
|
||
* bun scripts/push-satellites.ts --node fei-ios-01.shiro0.icu
|
||
* bun scripts/push-satellites.ts --force # swap NOW, aborting running tasks
|
||
*
|
||
* Scope defaults to the `default` cluster on purpose. The fleet also carries
|
||
* other teams' pools (`im-sg`, `pns`) whose nodes sit many commits back on their
|
||
* own cadence — sweeping them along with a routine deploy would be a far bigger
|
||
* change than the one being shipped, and nobody asked for it. `--all-clusters`
|
||
* is there when you do mean the whole fleet.
|
||
*
|
||
* Exit codes: 0 = every intended node is on (or armed for) the bundle;
|
||
* 1 = at least one push failed; 2 = bad usage / no bundle.
|
||
*/
|
||
import { existsSync, readFileSync } from "node:fs";
|
||
import { createHash } from "node:crypto";
|
||
import path from "node:path";
|
||
// Relative, not "@bbbug/shared/protocol": that alias only resolves from inside a
|
||
// workspace package (control-server/node_modules), not from scripts/. These are
|
||
// runtime VALUES, so unlike a type-only import it really has to resolve.
|
||
import {
|
||
SAT_API, BUNDLE_SHA256_HEADER, BUNDLE_VERSION_HEADER, BUNDLE_MODE_HEADER, CLUSTER_SECRET_HEADER,
|
||
} from "../shared/src/protocol";
|
||
import { DEFAULT_CLUSTER } from "../shared/src/types";
|
||
|
||
const ROOT = path.resolve(import.meta.dir, "..");
|
||
const BUNDLE_PATH = path.join(ROOT, "satellite/dist/satellite.js");
|
||
const VERSION_PATH = path.join(ROOT, "satellite/dist/VERSION");
|
||
const CONTROL_URL = (process.env.PUSH_SAT_CONTROL_URL || "http://127.0.0.1:5173").replace(/\/+$/, "");
|
||
const CLUSTER_SECRET = process.env.CLUSTER_SECRET || "";
|
||
|
||
interface Node {
|
||
id: string;
|
||
platform: string;
|
||
baseUrl: string;
|
||
version?: string | null;
|
||
live?: boolean;
|
||
lastSeenAt: number;
|
||
evalInFlight: number;
|
||
fixInFlight: number;
|
||
pendingDeploy?: string | null;
|
||
supportsQueuedDeploy?: boolean;
|
||
clusters?: string[] | null;
|
||
}
|
||
|
||
interface Opts {
|
||
force: boolean; dryRun: boolean; only: string[]; clusters: string[]; allClusters: boolean;
|
||
waitSec: number;
|
||
/** Versions whose bundle is byte-identical to this one — see --equivalent-version. */
|
||
equivalent: string[];
|
||
}
|
||
|
||
function usage(code: number): never {
|
||
console.log(`用法: bun scripts/push-satellites.ts [--force] [--dry-run] [--node <id>]... [--cluster <tag>]... [--all-clusters] [--wait-live <秒>]
|
||
|
||
默认: 只推 #${DEFAULT_CLUSTER} 池里落后的在线节点;忙节点走「空档部署」(不打断任务),
|
||
空闲节点立即换装。别的池(im-sg / pns 等)是别人按自己节奏在跑的,不顺手带上。
|
||
--force 立即换装,会中断正在跑的评估/修复。
|
||
--dry-run 只打印计划。
|
||
--node <id> 限定到具体节点(可重复),会绕过集群过滤。
|
||
--cluster <tag> 换一个集群(可重复)。
|
||
--all-clusters 整个 fleet,包括别的团队的池子。
|
||
--wait-live N 推之前等各节点心跳刷新的秒数(默认 45)。控制面刚重启时它的
|
||
节点能力位(supportsQueuedDeploy)要等下一次心跳才重新学到,
|
||
不等就会把忙节点误判成「不支持空档部署」。
|
||
--equivalent-version <sha>
|
||
跑着这些版本的节点视为「已是最新」(可重复)。bundle 的 VERSION
|
||
是 git SHA,所以只改了控制面/脚本的部署会产出一个逐字节相同、
|
||
只有版本号不同的 bundle;不加这个,每次部署都会让节点白白排空
|
||
队列重启一轮。deploy.sh 在新旧 bundle 的 sha256 相同时自动传。`);
|
||
process.exit(code);
|
||
}
|
||
|
||
function parseArgs(argv: string[]): Opts {
|
||
const o: Opts = { force: false, dryRun: false, only: [], clusters: [], allClusters: false, waitSec: 45, equivalent: [] };
|
||
for (let i = 0; i < argv.length; i++) {
|
||
const a = argv[i];
|
||
if (a === "--force") o.force = true;
|
||
else if (a === "--dry-run") o.dryRun = true;
|
||
else if (a === "--all-clusters") o.allClusters = true;
|
||
else if (a === "--node") { const v = argv[++i]; if (!v) usage(2); o.only.push(v); }
|
||
else if (a.startsWith("--node=")) o.only.push(a.slice("--node=".length));
|
||
else if (a === "--cluster") { const v = argv[++i]; if (!v) usage(2); o.clusters.push(v); }
|
||
else if (a.startsWith("--cluster=")) o.clusters.push(a.slice("--cluster=".length));
|
||
else if (a === "--wait-live") o.waitSec = Number(argv[++i] ?? "45");
|
||
else if (a.startsWith("--wait-live=")) o.waitSec = Number(a.slice("--wait-live=".length));
|
||
else if (a === "--equivalent-version") { const v = argv[++i]; if (!v) usage(2); o.equivalent.push(v); }
|
||
else if (a.startsWith("--equivalent-version=")) o.equivalent.push(a.slice("--equivalent-version=".length));
|
||
else if (a === "-h" || a === "--help") usage(0);
|
||
else { console.error(`未知参数: ${a}`); usage(2); }
|
||
}
|
||
if (!Number.isFinite(o.waitSec) || o.waitSec < 0) usage(2);
|
||
if (o.clusters.length === 0) o.clusters.push(DEFAULT_CLUSTER);
|
||
return o;
|
||
}
|
||
|
||
/** A node with no tags implicitly belongs to `default` — same rule the claim SQL applies. */
|
||
function clustersOf(n: Node): string[] {
|
||
const own = (n.clusters ?? []).filter(Boolean);
|
||
return own.length ? own : [DEFAULT_CLUSTER];
|
||
}
|
||
|
||
/** Is this node in scope? An explicit --node list wins over the cluster filter. */
|
||
function inScope(n: Node, o: Opts): boolean {
|
||
if (o.only.length) return o.only.includes(n.id);
|
||
if (o.allClusters) return true;
|
||
return clustersOf(n).some((c) => o.clusters.includes(c));
|
||
}
|
||
|
||
async function fetchNodes(): Promise<Node[]> {
|
||
const res = await fetch(`${CONTROL_URL}/api/nodes`, { signal: AbortSignal.timeout(15_000) });
|
||
if (!res.ok) throw new Error(`GET ${CONTROL_URL}/api/nodes → HTTP ${res.status}`);
|
||
const j = (await res.json()) as { nodes?: Node[] };
|
||
return j.nodes ?? [];
|
||
}
|
||
|
||
/**
|
||
* Wait until every live node has heartbeated since `since`.
|
||
*
|
||
* Not politeness — correctness. `supportsQueuedDeploy` and `pendingDeploy` are
|
||
* rebuilt from heartbeats in the control-server's memory, so in the seconds
|
||
* after a control-plane restart EVERY node looks like it can't do a queued
|
||
* deploy. Pushing then would refuse every busy node (or, with --force, kill its
|
||
* task). One heartbeat cycle (~10s) restores the truth.
|
||
*/
|
||
async function waitForFreshHeartbeats(since: number, opts: Opts): Promise<Node[]> {
|
||
const deadline = Date.now() + opts.waitSec * 1000;
|
||
let nodes = await fetchNodes();
|
||
if (opts.waitSec === 0) return nodes;
|
||
for (;;) {
|
||
const candidates = nodes.filter((n) => n.live && inScope(n, opts));
|
||
const stale = candidates.filter((n) => n.lastSeenAt < since);
|
||
if (candidates.length > 0 && stale.length === 0) return nodes;
|
||
if (Date.now() >= deadline) {
|
||
if (stale.length > 0) {
|
||
console.log(` ! 等心跳超时:${stale.map((n) => n.id).join(", ")} 仍是重启前的心跳,忙节点可能被判为不支持空档部署`);
|
||
}
|
||
return nodes;
|
||
}
|
||
await new Promise((r) => setTimeout(r, 3000));
|
||
nodes = await fetchNodes();
|
||
}
|
||
}
|
||
|
||
interface Outcome { node: string; state: "queued" | "swapped" | "current" | "armed" | "offline" | "failed"; note: string }
|
||
|
||
async function main(): Promise<void> {
|
||
const opts = parseArgs(process.argv.slice(2));
|
||
|
||
if (!existsSync(BUNDLE_PATH)) {
|
||
console.error(`没有 bundle:${BUNDLE_PATH}(先跑 bun run build:satellite)`);
|
||
process.exit(2);
|
||
}
|
||
const bytes = readFileSync(BUNDLE_PATH);
|
||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||
const version = existsSync(VERSION_PATH) ? readFileSync(VERSION_PATH, "utf8").trim() : "unknown";
|
||
// Same guard the deploy script applies: a truncated bundle passes the node's
|
||
// sha256 check (that only proves the bytes survived the wire) and then fails
|
||
// to boot, taking the node down until someone notices.
|
||
if (bytes.length < 100_000) {
|
||
console.error(`bundle 只有 ${bytes.length} 字节,像是被截断了,拒绝推送`);
|
||
process.exit(2);
|
||
}
|
||
if (!CLUSTER_SECRET) console.log("! CLUSTER_SECRET 为空 — 只有在节点也没配 secret 时才推得动");
|
||
|
||
console.log(`bundle ${version} · ${(bytes.length / 1024 / 1024).toFixed(2)} MB · sha256 ${sha256.slice(0, 12)}`);
|
||
|
||
const started = Date.now();
|
||
const nodes = await waitForFreshHeartbeats(started, opts);
|
||
const targets = nodes.filter((n) => inScope(n, opts));
|
||
for (const id of opts.only) {
|
||
if (!nodes.some((n) => n.id === id)) console.log(` ! 节点 ${id} 不在集群里`);
|
||
}
|
||
// Say what is being left alone, so a partial rollout never reads as a full one.
|
||
const skipped = nodes.filter((n) => !inScope(n, opts));
|
||
const scope = opts.only.length ? `节点 ${opts.only.join(", ")}`
|
||
: opts.allClusters ? "全部集群" : opts.clusters.map((c) => `#${c}`).join(" ");
|
||
console.log(`范围: ${scope} — ${targets.length} 台在列`
|
||
+ (skipped.length ? `,${skipped.length} 台不在范围内(${skipped.map((n) => n.id).join(", ")})` : ""));
|
||
if (targets.length === 0) { console.log("范围内没有节点"); return; }
|
||
|
||
const out: Outcome[] = [];
|
||
for (const n of targets) {
|
||
const tag = `${n.id} (${n.platform})`;
|
||
if (!n.live) { out.push({ node: n.id, state: "offline", note: "离线,本次跳过" }); console.log(` ⤬ ${tag} 离线`); continue; }
|
||
if (n.version === version) { out.push({ node: n.id, state: "current", note: "已是最新" }); console.log(` = ${tag} 已在 ${version}`); continue; }
|
||
// Same bytes, different stamp: pushing would drain and restart the node to
|
||
// change a version string. See --equivalent-version.
|
||
if (n.version && opts.equivalent.includes(n.version)) {
|
||
out.push({ node: n.id, state: "current", note: "字节相同" });
|
||
console.log(` = ${tag} 跑的是 ${n.version},与本 bundle 逐字节相同 — 不推`);
|
||
continue;
|
||
}
|
||
// Already armed with these bytes (under this stamp or a byte-identical one):
|
||
// re-arming would only restamp a rollout that is already on its way.
|
||
if (n.pendingDeploy && (n.pendingDeploy === version || opts.equivalent.includes(n.pendingDeploy))) {
|
||
out.push({ node: n.id, state: "armed", note: "已排队" });
|
||
console.log(` = ${tag} 已排队 ${n.pendingDeploy},等它自己换装`);
|
||
continue;
|
||
}
|
||
|
||
const busy = n.evalInFlight > 0 || n.fixInFlight > 0;
|
||
// queued wins over force, exactly as the route decides it: asking to wait
|
||
// and to interrupt at once is contradictory, and waiting is the safe read.
|
||
const wantQueued = busy && !opts.force;
|
||
if (wantQueued && !n.supportsQueuedDeploy) {
|
||
const note = "节点忙,且其卫星版本不支持空档部署 — 等它空闲后重跑,或用 --force(会中断任务)";
|
||
out.push({ node: n.id, state: "failed", note });
|
||
console.log(` ✗ ${tag} ${note}`);
|
||
continue;
|
||
}
|
||
const mode = wantQueued ? "queued" : "now";
|
||
const what = `${n.version ?? "unknown"} → ${version}${busy ? (wantQueued ? " [空档部署]" : " [强制,中断任务]") : ""}`;
|
||
if (opts.dryRun) { console.log(` · ${tag} 将推送 ${what}`); continue; }
|
||
|
||
try {
|
||
const res = await fetch(n.baseUrl.replace(/\/+$/, "") + SAT_API.deploy, {
|
||
method: "POST",
|
||
body: bytes,
|
||
headers: {
|
||
"content-type": "application/octet-stream",
|
||
[BUNDLE_SHA256_HEADER]: sha256,
|
||
[BUNDLE_VERSION_HEADER]: version,
|
||
// The node re-decides now-vs-arm on arrival: it knows whether it is
|
||
// busy at the instant the bytes land, which beats our heartbeat view.
|
||
[BUNDLE_MODE_HEADER]: mode,
|
||
"x-deploy-by": process.env.PUSH_SAT_BY || "deploy.sh",
|
||
...(CLUSTER_SECRET ? { [CLUSTER_SECRET_HEADER]: CLUSTER_SECRET } : {}),
|
||
},
|
||
signal: AbortSignal.timeout(120_000),
|
||
});
|
||
const j = (await res.json().catch(() => ({}))) as
|
||
{ ok?: boolean; error?: string; skipped?: boolean; queued?: boolean; busy?: string };
|
||
if (!res.ok || !j.ok) {
|
||
const note = j.error || `HTTP ${res.status}`;
|
||
out.push({ node: n.id, state: "failed", note });
|
||
console.log(` ✗ ${tag} ${note}`);
|
||
} else if (j.queued) {
|
||
out.push({ node: n.id, state: "queued", note: j.busy || "忙" });
|
||
console.log(` ⏸ ${tag} 已排队(${j.busy || "忙"}) — 当前任务结束后自动换装重启`);
|
||
} else if (j.skipped) {
|
||
out.push({ node: n.id, state: "current", note: "已是最新" });
|
||
console.log(` = ${tag} 已是最新`);
|
||
} else {
|
||
out.push({ node: n.id, state: "swapped", note: what });
|
||
console.log(` ✓ ${tag} 已换装 ${what},重启中`);
|
||
}
|
||
} catch (e) {
|
||
const note = (e as Error).message;
|
||
out.push({ node: n.id, state: "failed", note });
|
||
console.log(` ✗ ${tag} ${note}`);
|
||
}
|
||
}
|
||
|
||
if (opts.dryRun) { console.log("(--dry-run,什么都没改)"); return; }
|
||
|
||
const n = (s: Outcome["state"]) => out.filter((o) => o.state === s).length;
|
||
const parts = [
|
||
n("swapped") ? `${n("swapped")} 台已换装重启` : "",
|
||
n("queued") ? `${n("queued")} 台已排队(空档换装)` : "",
|
||
n("current") + n("armed") ? `${n("current") + n("armed")} 台无需动作` : "",
|
||
n("offline") ? `${n("offline")} 台离线` : "",
|
||
n("failed") ? `${n("failed")} 台失败` : "",
|
||
].filter(Boolean);
|
||
console.log(`卫星: ${parts.join(" · ") || "无节点"}`);
|
||
// An armed node is NOT yet running the new code — say so, or a green deploy
|
||
// reads as "rolled out" while a busy node still runs the old bundle for hours.
|
||
if (n("queued")) console.log(" 排队的节点仍在跑旧代码,直到它当前的任务结束;面板上显示「已排队」。");
|
||
if (n("offline")) console.log(" 离线节点回来后不会自动补装 — 重跑本脚本或在面板上推。");
|
||
if (n("failed")) process.exit(1);
|
||
}
|
||
|
||
main().catch((e) => { console.error(`卫星推送失败: ${(e as Error).message}`); process.exit(1); });
|