Files
streamline/scripts/check-node-health.ts
2026-11-16 17:26:44 -08:00

188 lines
8.8 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* End-to-end smoke test for 节点健康, runnable on a laptop with nothing else set up.
*
* bun scripts/check-node-health.ts
*
* What it actually exercises:
* - the REAL satellite sampler (satellite/src/metrics.ts) against this host
* - the REAL control-plane rule engine (control-server/src/node_health.ts)
* - the wire shape between them, over real HTTP
*
* What it fakes: only the transport ends. A stub control plane serves the two
* internal endpoints the satellite calls, and the satellite side is driven
* directly rather than by booting the whole process (which would want a
* workspace, adb/xcrun, and a live claim loop).
*
* It touches NOTHING in production: DATABASE_URL is forced at a closed local
* port, and no Lark chat is configured so the alert path is a no-op. Safe to run
* on a developer machine at any time.
*/
// Must precede any import that pulls in the DAL — imports are hoisted, so this
// has to be its own statement before the rest (see control-server/test/setup_env.ts).
process.env.DATABASE_URL ||= "postgres://test:test@127.0.0.1:1/streamline_test";
delete process.env.LARK_NOTIFY_CHAT_ID;
// Short windows so the run demonstrates firing in seconds instead of minutes.
process.env.HEALTH_SUSTAIN_MS ||= "1000";
process.env.HEALTH_METRICS_STALE_MS ||= "180000";
// Fast sampling so cpuPct has a baseline to diff against within this run — at
// the production 15s cadence a 2-second script would only ever see one tick.
process.env.METRICS_SAMPLE_MS ||= "700";
const { startMetricsSampler, readMetrics } = await import("../satellite/src/metrics");
const { noteNodeMetrics, judgeNode, __resetHealthState } = await import("../control-server/src/node_health");
// Relative, not "@bbbug/shared/*": that alias only resolves from inside a
// workspace package, not from scripts/ (same reason push-satellites.ts does it).
import type { HeartbeatPayload, NodeMetrics } from "../shared/src/protocol";
import type { ClusterNode } from "../shared/src/types";
const ok = (s: string) => console.log(` \x1b[32m✓\x1b[0m ${s}`);
const info = (s: string) => console.log(` \x1b[2m${s}\x1b[0m`);
const head = (s: string) => console.log(`\n\x1b[1m${s}\x1b[0m`);
let failures = 0;
const check = (cond: boolean, s: string) => {
if (cond) ok(s);
else { failures++; console.log(` \x1b[31m✗ ${s}\x1b[0m`); }
};
// ---------------------------------------------------------------------------
head("1. 采集器:真实采样这台机器");
startMetricsSampler();
// Give the slow tier (vm_stat / sysctl / claude auth status) a moment to land.
await Bun.sleep(2000);
const m = readMetrics();
if (!m) { console.error("采样器没有产出快照"); process.exit(1); }
info(`CPU ${m.cpuPct ?? "—"}% 负载 ${m.loadPct}% ${m.cores} 核`);
info(`内存 ${m.memUsedPct ?? "—"}% 共 ${(m.memTotalBytes / 2 ** 30).toFixed(0)}GB swap ${((m.swapUsedBytes ?? 0) / 2 ** 30).toFixed(1)}GB`);
info(`磁盘 ${m.diskUsedPct ?? "—"}% 剩余 ${((m.diskFreeBytes ?? 0) / 2 ** 30).toFixed(0)}GB`);
info(`Agent ${m.agent.mode} · loggedIn=${m.agent.loggedIn} · ${m.agent.authMethod ?? "—"}`);
info(`慢档耗时 ${m.slowMs}ms${m.slowError ? ` · 错误 ${m.slowError}` : ""}`);
check(m.diskUsedPct !== null, "磁盘已采集");
check(m.memUsedPct !== null, "内存已采集(macOS 走 vm_stat,非 os.freemem)");
check(m.memUsedPct === null || m.memUsedPct < 97, "内存读数不是 os.freemem 的假高值(应远低于 97%)");
// ---------------------------------------------------------------------------
head("2. 心跳路径:必须零 I/O");
const t0 = performance.now();
for (let i = 0; i < 200_000; i++) readMetrics();
const perCall = ((performance.now() - t0) * 1000) / 200_000;
info(`readMetrics() × 200k → ${perCall.toFixed(4)} µs/次`);
check(perCall < 1, "心跳读快照是纯属性读取(< 1µs)");
// ---------------------------------------------------------------------------
head("3. 传输:真实 HTTP,校验 wire shape");
let received: HeartbeatPayload | null = null;
const server = Bun.serve({
port: 0,
async fetch(req) {
const url = new URL(req.url);
if (url.pathname === "/api/internal/nodes/heartbeat") {
received = (await req.json()) as HeartbeatPayload;
return Response.json({ ok: true });
}
return Response.json({ ok: true, nodeId: "test" });
},
});
const hb: HeartbeatPayload = {
nodeId: "test-node", evalInFlight: 0, evalCapacity: 3, fixInFlight: 0, fixCapacity: 1,
metrics: readMetrics(), pendingVersion: null, queuedDeploy: true,
};
const res = await fetch(`http://127.0.0.1:${server.port}/api/internal/nodes/heartbeat`, {
method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(hb),
});
check(res.ok, "心跳 POST 成功");
check(received !== null && received!.metrics !== null, "metrics 字段完整穿过 JSON 序列化");
check(
received?.metrics?.agent !== undefined && typeof received?.metrics?.agent.authGateMs === "number",
"AgentHealth 嵌套结构完整",
);
const wireBytes = JSON.stringify(hb).length;
info(`心跳报文 ${wireBytes} 字节(加指标前约 ${JSON.stringify({ ...hb, metrics: null }).length} 字节)`);
check(wireBytes < 2000, "报文仍然很小(< 2KB)");
server.stop();
// ---------------------------------------------------------------------------
head("4. 规则引擎:阈值、迟滞、冷却");
__resetHealthState();
const NOW = Date.now();
const node = (over: Partial<ClusterNode> = {}): ClusterNode => ({
id: "test-node", baseUrl: "http://x", platform: "ios", status: "online",
lastSeenAt: NOW, evalInFlight: 0, evalCapacity: 3, fixInFlight: 0, fixCapacity: 1,
version: "v", createdAt: 0, live: true, ...over,
} as ClusterNode);
const withDisk = (pct: number, over: Partial<NodeMetrics> = {}): NodeMetrics =>
({ ...readMetrics()!, at: Date.now(), diskUsedPct: pct, ...over });
noteNodeMetrics("test-node", withDisk(50));
check(judgeNode(node(), NOW).health.level === "ok", "50% 磁盘 → 正常");
noteNodeMetrics("test-node", withDisk(93));
const spike = judgeNode(node(), NOW);
check(spike.health.level === "critical", "93% 磁盘 → 面板立即标红");
check(spike.toNotify.length === 0, "但未达持续时长,不推送飞书(避免为瞬时尖峰打扰)");
noteNodeMetrics("test-node", withDisk(93));
const sustained = judgeNode(node(), NOW + 5000);
check(sustained.toNotify.length === 1, "持续超过窗口 → 推送一次");
info(`告警文案:${sustained.toNotify[0]?.message}`);
noteNodeMetrics("test-node", withDisk(93));
check(judgeNode(node(), NOW + 6000).toNotify.length === 0, "冷却期内不重复推送");
noteNodeMetrics("test-node", withDisk(87));
check(judgeNode(node(), NOW + 7000).health.level === "critical", "回落到 87%(迟滞带内)仍保持告警");
noteNodeMetrics("test-node", withDisk(84));
check(judgeNode(node(), NOW + 8000).health.level === "ok", "回落到 84%(低于清除线)→ 恢复");
__resetHealthState();
noteNodeMetrics("test-node", withDisk(50, { agent: { ...m.agent, loggedIn: false } }));
judgeNode(node(), NOW);
const agentAlert = judgeNode(node(), NOW + 5000);
check(agentAlert.toNotify.some((a) => a.rule === "agent"), "claude 未登录 → 告警");
info(`告警文案:${agentAlert.toNotify.find((a) => a.rule === "agent")?.message}`);
__resetHealthState();
noteNodeMetrics("test-node", withDisk(50, { agent: { ...m.agent, mode: "cursor", loggedIn: null } }));
judgeNode(node(), NOW);
check(
judgeNode(node(), NOW + 5000).health.alerts.every((a) => a.rule !== "agent"),
"cursor 引擎节点(loggedIn=null)默认通过,不误报",
);
__resetHealthState();
judgeNode(node({ live: false, lastSeenAt: NOW - 90_000 }), NOW);
const off = judgeNode(node({ live: false, lastSeenAt: NOW - 90_000 }), NOW + 5000);
check(off.toNotify.some((a) => a.rule === "offline"), "节点离线 → 告警");
check(off.health.alerts.length === 1, "离线时短路其余规则(不用陈旧数据判断磁盘)");
// ---------------------------------------------------------------------------
head("5. 容错:采集失败不能变成「一切正常」");
__resetHealthState();
noteNodeMetrics("test-node", withDisk(50, { diskUsedPct: null, diskFreeBytes: null }));
const unknown = judgeNode(node(), NOW);
check(unknown.health.alerts.every((a) => a.rule !== "disk"), "磁盘未知 → 不误报磁盘告警");
info("(面板此时显示「未知」而非 0%,不会被读成「空间充足」)");
__resetHealthState();
noteNodeMetrics("test-node", withDisk(50, { at: Date.now() - 600_000 }));
judgeNode(node(), NOW);
const stale = judgeNode(node(), NOW + 5000);
check(stale.health.alerts.some((a) => a.rule === "metrics_stale"), "心跳正常但采集器停摆 → 指标停更告警");
// ---------------------------------------------------------------------------
console.log(
failures === 0
? `\n\x1b[32m全部通过\x1b[0m — 采集、传输、判定、容错四段链路都验证过了。`
: `\n\x1b[31m${failures} 项失败\x1b[0m`,
);
process.exit(failures === 0 ? 0 : 1);