Files
streamline/scripts/push-satellites.ts
T
2026-11-16 17:26:44 -08:00

307 lines
16 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bun
/**
* 卫星热部署 CLI — the non-interactive twin of 管理 → 卫星热部署.
*
* Exists so a deploy can finish the job: `deploy.sh` builds the bundle, then
* calls this to actually get it onto the nodes, instead of ending with a note
* telling a human to go click three buttons (which is how nodes ended up
* running week-old code while the panel cheerfully said 「最新」).
*
* Why it is safe to automate, when the panel deliberately kept it manual:
* the panel's danger is 强制部署 — swapping the bundle exits the satellite
* process and kills whatever it was running. This script NEVER does that by
* default. A busy node gets 「空档部署」: the node persists the bundle, stops
* claiming new work, finishes its current task, and swaps in the gap. Nothing
* is interrupted, so there is no judgement call left for a human to make.
* `--force` restores the destructive behaviour and is opt-in only.
*
* Why it talks HTTP instead of importing the control plane's own route:
* - `POST /api/deploy` is SSO+admin gated (browser credentials) — a script
* has none, and weakening that gate to let scripts in would be a far worse
* trade than this file.
* - `pendingDeploy` / `supportsQueuedDeploy` live in the control-server's
* MEMORY (learned from heartbeats, see cluster.ts), not the DB, so a
* separate process must ask the running server for them — `GET /api/nodes`
* is read-only and unauthenticated.
* So: read node state from the control plane, push bytes straight to each node
* over the same cluster-secret channel the control plane itself uses.
*
* Run it ON the control-plane box (it needs CLUSTER_SECRET from .env, the built
* bundle, and network to the nodes):
*
* bun scripts/push-satellites.ts # push to the `default` pool
* bun scripts/push-satellites.ts --dry-run # print the plan, touch nothing
* bun scripts/push-satellites.ts --all-clusters # every pool, incl. other teams'
* bun scripts/push-satellites.ts --cluster pns
* bun scripts/push-satellites.ts --node fei-ios-01.shiro0.icu
* bun scripts/push-satellites.ts --force # swap NOW, aborting running tasks
*
* Scope defaults to the `default` cluster on purpose. The fleet also carries
* other teams' pools (`im-sg`, `pns`) whose nodes sit many commits back on their
* own cadence — sweeping them along with a routine deploy would be a far bigger
* change than the one being shipped, and nobody asked for it. `--all-clusters`
* is there when you do mean the whole fleet.
*
* Exit codes: 0 = every intended node is on (or armed for) the bundle;
* 1 = at least one push failed; 2 = bad usage / no bundle.
*/
import { existsSync, readFileSync } from "node:fs";
import { createHash } from "node:crypto";
import path from "node:path";
// Relative, not "@bbbug/shared/protocol": that alias only resolves from inside a
// workspace package (control-server/node_modules), not from scripts/. These are
// runtime VALUES, so unlike a type-only import it really has to resolve.
import {
SAT_API, BUNDLE_SHA256_HEADER, BUNDLE_VERSION_HEADER, BUNDLE_MODE_HEADER, CLUSTER_SECRET_HEADER,
} from "../shared/src/protocol";
import { DEFAULT_CLUSTER } from "../shared/src/types";
const ROOT = path.resolve(import.meta.dir, "..");
const BUNDLE_PATH = path.join(ROOT, "satellite/dist/satellite.js");
const VERSION_PATH = path.join(ROOT, "satellite/dist/VERSION");
const CONTROL_URL = (process.env.PUSH_SAT_CONTROL_URL || "http://127.0.0.1:5173").replace(/\/+$/, "");
const CLUSTER_SECRET = process.env.CLUSTER_SECRET || "";
interface Node {
id: string;
platform: string;
baseUrl: string;
version?: string | null;
live?: boolean;
lastSeenAt: number;
evalInFlight: number;
fixInFlight: number;
pendingDeploy?: string | null;
supportsQueuedDeploy?: boolean;
clusters?: string[] | null;
}
interface Opts {
force: boolean; dryRun: boolean; only: string[]; clusters: string[]; allClusters: boolean;
waitSec: number;
/** Versions whose bundle is byte-identical to this one — see --equivalent-version. */
equivalent: string[];
}
function usage(code: number): never {
console.log(`用法: bun scripts/push-satellites.ts [--force] [--dry-run] [--node <id>]... [--cluster <tag>]... [--all-clusters] [--wait-live <秒>]
默认: 只推 #${DEFAULT_CLUSTER} 池里落后的在线节点;忙节点走「空档部署」(不打断任务),
空闲节点立即换装。别的池(im-sg / pns 等)是别人按自己节奏在跑的,不顺手带上。
--force 立即换装,会中断正在跑的评估/修复。
--dry-run 只打印计划。
--node <id> 限定到具体节点(可重复),会绕过集群过滤。
--cluster <tag> 换一个集群(可重复)。
--all-clusters 整个 fleet,包括别的团队的池子。
--wait-live N 推之前等各节点心跳刷新的秒数(默认 45)。控制面刚重启时它的
节点能力位(supportsQueuedDeploy)要等下一次心跳才重新学到,
不等就会把忙节点误判成「不支持空档部署」。
--equivalent-version <sha>
跑着这些版本的节点视为「已是最新」(可重复)。bundle 的 VERSION
是 git SHA,所以只改了控制面/脚本的部署会产出一个逐字节相同、
只有版本号不同的 bundle;不加这个,每次部署都会让节点白白排空
队列重启一轮。deploy.sh 在新旧 bundle 的 sha256 相同时自动传。`);
process.exit(code);
}
function parseArgs(argv: string[]): Opts {
const o: Opts = { force: false, dryRun: false, only: [], clusters: [], allClusters: false, waitSec: 45, equivalent: [] };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a === "--force") o.force = true;
else if (a === "--dry-run") o.dryRun = true;
else if (a === "--all-clusters") o.allClusters = true;
else if (a === "--node") { const v = argv[++i]; if (!v) usage(2); o.only.push(v); }
else if (a.startsWith("--node=")) o.only.push(a.slice("--node=".length));
else if (a === "--cluster") { const v = argv[++i]; if (!v) usage(2); o.clusters.push(v); }
else if (a.startsWith("--cluster=")) o.clusters.push(a.slice("--cluster=".length));
else if (a === "--wait-live") o.waitSec = Number(argv[++i] ?? "45");
else if (a.startsWith("--wait-live=")) o.waitSec = Number(a.slice("--wait-live=".length));
else if (a === "--equivalent-version") { const v = argv[++i]; if (!v) usage(2); o.equivalent.push(v); }
else if (a.startsWith("--equivalent-version=")) o.equivalent.push(a.slice("--equivalent-version=".length));
else if (a === "-h" || a === "--help") usage(0);
else { console.error(`未知参数: ${a}`); usage(2); }
}
if (!Number.isFinite(o.waitSec) || o.waitSec < 0) usage(2);
if (o.clusters.length === 0) o.clusters.push(DEFAULT_CLUSTER);
return o;
}
/** A node with no tags implicitly belongs to `default` — same rule the claim SQL applies. */
function clustersOf(n: Node): string[] {
const own = (n.clusters ?? []).filter(Boolean);
return own.length ? own : [DEFAULT_CLUSTER];
}
/** Is this node in scope? An explicit --node list wins over the cluster filter. */
function inScope(n: Node, o: Opts): boolean {
if (o.only.length) return o.only.includes(n.id);
if (o.allClusters) return true;
return clustersOf(n).some((c) => o.clusters.includes(c));
}
async function fetchNodes(): Promise<Node[]> {
const res = await fetch(`${CONTROL_URL}/api/nodes`, { signal: AbortSignal.timeout(15_000) });
if (!res.ok) throw new Error(`GET ${CONTROL_URL}/api/nodes → HTTP ${res.status}`);
const j = (await res.json()) as { nodes?: Node[] };
return j.nodes ?? [];
}
/**
* Wait until every live node has heartbeated since `since`.
*
* Not politeness — correctness. `supportsQueuedDeploy` and `pendingDeploy` are
* rebuilt from heartbeats in the control-server's memory, so in the seconds
* after a control-plane restart EVERY node looks like it can't do a queued
* deploy. Pushing then would refuse every busy node (or, with --force, kill its
* task). One heartbeat cycle (~10s) restores the truth.
*/
async function waitForFreshHeartbeats(since: number, opts: Opts): Promise<Node[]> {
const deadline = Date.now() + opts.waitSec * 1000;
let nodes = await fetchNodes();
if (opts.waitSec === 0) return nodes;
for (;;) {
const candidates = nodes.filter((n) => n.live && inScope(n, opts));
const stale = candidates.filter((n) => n.lastSeenAt < since);
if (candidates.length > 0 && stale.length === 0) return nodes;
if (Date.now() >= deadline) {
if (stale.length > 0) {
console.log(` ! 等心跳超时:${stale.map((n) => n.id).join(", ")} 仍是重启前的心跳,忙节点可能被判为不支持空档部署`);
}
return nodes;
}
await new Promise((r) => setTimeout(r, 3000));
nodes = await fetchNodes();
}
}
interface Outcome { node: string; state: "queued" | "swapped" | "current" | "armed" | "offline" | "failed"; note: string }
async function main(): Promise<void> {
const opts = parseArgs(process.argv.slice(2));
if (!existsSync(BUNDLE_PATH)) {
console.error(`没有 bundle:${BUNDLE_PATH}(先跑 bun run build:satellite)`);
process.exit(2);
}
const bytes = readFileSync(BUNDLE_PATH);
const sha256 = createHash("sha256").update(bytes).digest("hex");
const version = existsSync(VERSION_PATH) ? readFileSync(VERSION_PATH, "utf8").trim() : "unknown";
// Same guard the deploy script applies: a truncated bundle passes the node's
// sha256 check (that only proves the bytes survived the wire) and then fails
// to boot, taking the node down until someone notices.
if (bytes.length < 100_000) {
console.error(`bundle 只有 ${bytes.length} 字节,像是被截断了,拒绝推送`);
process.exit(2);
}
if (!CLUSTER_SECRET) console.log("! CLUSTER_SECRET 为空 — 只有在节点也没配 secret 时才推得动");
console.log(`bundle ${version} · ${(bytes.length / 1024 / 1024).toFixed(2)} MB · sha256 ${sha256.slice(0, 12)}`);
const started = Date.now();
const nodes = await waitForFreshHeartbeats(started, opts);
const targets = nodes.filter((n) => inScope(n, opts));
for (const id of opts.only) {
if (!nodes.some((n) => n.id === id)) console.log(` ! 节点 ${id} 不在集群里`);
}
// Say what is being left alone, so a partial rollout never reads as a full one.
const skipped = nodes.filter((n) => !inScope(n, opts));
const scope = opts.only.length ? `节点 ${opts.only.join(", ")}`
: opts.allClusters ? "全部集群" : opts.clusters.map((c) => `#${c}`).join(" ");
console.log(`范围: ${scope} — ${targets.length} 台在列`
+ (skipped.length ? `,${skipped.length} 台不在范围内(${skipped.map((n) => n.id).join(", ")})` : ""));
if (targets.length === 0) { console.log("范围内没有节点"); return; }
const out: Outcome[] = [];
for (const n of targets) {
const tag = `${n.id} (${n.platform})`;
if (!n.live) { out.push({ node: n.id, state: "offline", note: "离线,本次跳过" }); console.log(` ⤬ ${tag} 离线`); continue; }
if (n.version === version) { out.push({ node: n.id, state: "current", note: "已是最新" }); console.log(` = ${tag} 已在 ${version}`); continue; }
// Same bytes, different stamp: pushing would drain and restart the node to
// change a version string. See --equivalent-version.
if (n.version && opts.equivalent.includes(n.version)) {
out.push({ node: n.id, state: "current", note: "字节相同" });
console.log(` = ${tag} 跑的是 ${n.version},与本 bundle 逐字节相同 — 不推`);
continue;
}
// Already armed with these bytes (under this stamp or a byte-identical one):
// re-arming would only restamp a rollout that is already on its way.
if (n.pendingDeploy && (n.pendingDeploy === version || opts.equivalent.includes(n.pendingDeploy))) {
out.push({ node: n.id, state: "armed", note: "已排队" });
console.log(` = ${tag} 已排队 ${n.pendingDeploy},等它自己换装`);
continue;
}
const busy = n.evalInFlight > 0 || n.fixInFlight > 0;
// queued wins over force, exactly as the route decides it: asking to wait
// and to interrupt at once is contradictory, and waiting is the safe read.
const wantQueued = busy && !opts.force;
if (wantQueued && !n.supportsQueuedDeploy) {
const note = "节点忙,且其卫星版本不支持空档部署 — 等它空闲后重跑,或用 --force(会中断任务)";
out.push({ node: n.id, state: "failed", note });
console.log(` ✗ ${tag} ${note}`);
continue;
}
const mode = wantQueued ? "queued" : "now";
const what = `${n.version ?? "unknown"} → ${version}${busy ? (wantQueued ? " [空档部署]" : " [强制,中断任务]") : ""}`;
if (opts.dryRun) { console.log(` · ${tag} 将推送 ${what}`); continue; }
try {
const res = await fetch(n.baseUrl.replace(/\/+$/, "") + SAT_API.deploy, {
method: "POST",
body: bytes,
headers: {
"content-type": "application/octet-stream",
[BUNDLE_SHA256_HEADER]: sha256,
[BUNDLE_VERSION_HEADER]: version,
// The node re-decides now-vs-arm on arrival: it knows whether it is
// busy at the instant the bytes land, which beats our heartbeat view.
[BUNDLE_MODE_HEADER]: mode,
"x-deploy-by": process.env.PUSH_SAT_BY || "deploy.sh",
...(CLUSTER_SECRET ? { [CLUSTER_SECRET_HEADER]: CLUSTER_SECRET } : {}),
},
signal: AbortSignal.timeout(120_000),
});
const j = (await res.json().catch(() => ({}))) as
{ ok?: boolean; error?: string; skipped?: boolean; queued?: boolean; busy?: string };
if (!res.ok || !j.ok) {
const note = j.error || `HTTP ${res.status}`;
out.push({ node: n.id, state: "failed", note });
console.log(` ✗ ${tag} ${note}`);
} else if (j.queued) {
out.push({ node: n.id, state: "queued", note: j.busy || "忙" });
console.log(` ⏸ ${tag} 已排队(${j.busy || "忙"}) — 当前任务结束后自动换装重启`);
} else if (j.skipped) {
out.push({ node: n.id, state: "current", note: "已是最新" });
console.log(` = ${tag} 已是最新`);
} else {
out.push({ node: n.id, state: "swapped", note: what });
console.log(` ✓ ${tag} 已换装 ${what},重启中`);
}
} catch (e) {
const note = (e as Error).message;
out.push({ node: n.id, state: "failed", note });
console.log(` ✗ ${tag} ${note}`);
}
}
if (opts.dryRun) { console.log("(--dry-run,什么都没改)"); return; }
const n = (s: Outcome["state"]) => out.filter((o) => o.state === s).length;
const parts = [
n("swapped") ? `${n("swapped")} 台已换装重启` : "",
n("queued") ? `${n("queued")} 台已排队(空档换装)` : "",
n("current") + n("armed") ? `${n("current") + n("armed")} 台无需动作` : "",
n("offline") ? `${n("offline")} 台离线` : "",
n("failed") ? `${n("failed")} 台失败` : "",
].filter(Boolean);
console.log(`卫星: ${parts.join(" · ") || "无节点"}`);
// An armed node is NOT yet running the new code — say so, or a green deploy
// reads as "rolled out" while a busy node still runs the old bundle for hours.
if (n("queued")) console.log(" 排队的节点仍在跑旧代码,直到它当前的任务结束;面板上显示「已排队」。");
if (n("offline")) console.log(" 离线节点回来后不会自动补装 — 重跑本脚本或在面板上推。");
if (n("failed")) process.exit(1);
}
main().catch((e) => { console.error(`卫星推送失败: ${(e as Error).message}`); process.exit(1); });