188 lines
8.8 KiB
TypeScript
188 lines
8.8 KiB
TypeScript
/**
|
||
* End-to-end smoke test for 节点健康, runnable on a laptop with nothing else set up.
|
||
*
|
||
* bun scripts/check-node-health.ts
|
||
*
|
||
* What it actually exercises:
|
||
* - the REAL satellite sampler (satellite/src/metrics.ts) against this host
|
||
* - the REAL control-plane rule engine (control-server/src/node_health.ts)
|
||
* - the wire shape between them, over real HTTP
|
||
*
|
||
* What it fakes: only the transport ends. A stub control plane serves the two
|
||
* internal endpoints the satellite calls, and the satellite side is driven
|
||
* directly rather than by booting the whole process (which would want a
|
||
* workspace, adb/xcrun, and a live claim loop).
|
||
*
|
||
* It touches NOTHING in production: DATABASE_URL is forced at a closed local
|
||
* port, and no Lark chat is configured so the alert path is a no-op. Safe to run
|
||
* on a developer machine at any time.
|
||
*/
|
||
|
||
// Must precede any import that pulls in the DAL — imports are hoisted, so this
|
||
// has to be its own statement before the rest (see control-server/test/setup_env.ts).
|
||
process.env.DATABASE_URL ||= "postgres://test:test@127.0.0.1:1/streamline_test";
|
||
delete process.env.LARK_NOTIFY_CHAT_ID;
|
||
// Short windows so the run demonstrates firing in seconds instead of minutes.
|
||
process.env.HEALTH_SUSTAIN_MS ||= "1000";
|
||
process.env.HEALTH_METRICS_STALE_MS ||= "180000";
|
||
// Fast sampling so cpuPct has a baseline to diff against within this run — at
|
||
// the production 15s cadence a 2-second script would only ever see one tick.
|
||
process.env.METRICS_SAMPLE_MS ||= "700";
|
||
|
||
const { startMetricsSampler, readMetrics } = await import("../satellite/src/metrics");
|
||
const { noteNodeMetrics, judgeNode, __resetHealthState } = await import("../control-server/src/node_health");
|
||
// Relative, not "@bbbug/shared/*": that alias only resolves from inside a
|
||
// workspace package, not from scripts/ (same reason push-satellites.ts does it).
|
||
import type { HeartbeatPayload, NodeMetrics } from "../shared/src/protocol";
|
||
import type { ClusterNode } from "../shared/src/types";
|
||
|
||
const ok = (s: string) => console.log(` \x1b[32m✓\x1b[0m ${s}`);
|
||
const info = (s: string) => console.log(` \x1b[2m${s}\x1b[0m`);
|
||
const head = (s: string) => console.log(`\n\x1b[1m${s}\x1b[0m`);
|
||
let failures = 0;
|
||
const check = (cond: boolean, s: string) => {
|
||
if (cond) ok(s);
|
||
else { failures++; console.log(` \x1b[31m✗ ${s}\x1b[0m`); }
|
||
};
|
||
|
||
// ---------------------------------------------------------------------------
|
||
head("1. 采集器:真实采样这台机器");
|
||
|
||
startMetricsSampler();
|
||
// Give the slow tier (vm_stat / sysctl / claude auth status) a moment to land.
|
||
await Bun.sleep(2000);
|
||
const m = readMetrics();
|
||
if (!m) { console.error("采样器没有产出快照"); process.exit(1); }
|
||
|
||
info(`CPU ${m.cpuPct ?? "—"}% 负载 ${m.loadPct}% ${m.cores} 核`);
|
||
info(`内存 ${m.memUsedPct ?? "—"}% 共 ${(m.memTotalBytes / 2 ** 30).toFixed(0)}GB swap ${((m.swapUsedBytes ?? 0) / 2 ** 30).toFixed(1)}GB`);
|
||
info(`磁盘 ${m.diskUsedPct ?? "—"}% 剩余 ${((m.diskFreeBytes ?? 0) / 2 ** 30).toFixed(0)}GB`);
|
||
info(`Agent ${m.agent.mode} · loggedIn=${m.agent.loggedIn} · ${m.agent.authMethod ?? "—"}`);
|
||
info(`慢档耗时 ${m.slowMs}ms${m.slowError ? ` · 错误 ${m.slowError}` : ""}`);
|
||
|
||
check(m.diskUsedPct !== null, "磁盘已采集");
|
||
check(m.memUsedPct !== null, "内存已采集(macOS 走 vm_stat,非 os.freemem)");
|
||
check(m.memUsedPct === null || m.memUsedPct < 97, "内存读数不是 os.freemem 的假高值(应远低于 97%)");
|
||
|
||
// ---------------------------------------------------------------------------
|
||
head("2. 心跳路径:必须零 I/O");
|
||
|
||
const t0 = performance.now();
|
||
for (let i = 0; i < 200_000; i++) readMetrics();
|
||
const perCall = ((performance.now() - t0) * 1000) / 200_000;
|
||
info(`readMetrics() × 200k → ${perCall.toFixed(4)} µs/次`);
|
||
check(perCall < 1, "心跳读快照是纯属性读取(< 1µs)");
|
||
|
||
// ---------------------------------------------------------------------------
|
||
head("3. 传输:真实 HTTP,校验 wire shape");
|
||
|
||
let received: HeartbeatPayload | null = null;
|
||
const server = Bun.serve({
|
||
port: 0,
|
||
async fetch(req) {
|
||
const url = new URL(req.url);
|
||
if (url.pathname === "/api/internal/nodes/heartbeat") {
|
||
received = (await req.json()) as HeartbeatPayload;
|
||
return Response.json({ ok: true });
|
||
}
|
||
return Response.json({ ok: true, nodeId: "test" });
|
||
},
|
||
});
|
||
|
||
const hb: HeartbeatPayload = {
|
||
nodeId: "test-node", evalInFlight: 0, evalCapacity: 3, fixInFlight: 0, fixCapacity: 1,
|
||
metrics: readMetrics(), pendingVersion: null, queuedDeploy: true,
|
||
};
|
||
const res = await fetch(`http://127.0.0.1:${server.port}/api/internal/nodes/heartbeat`, {
|
||
method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(hb),
|
||
});
|
||
check(res.ok, "心跳 POST 成功");
|
||
check(received !== null && received!.metrics !== null, "metrics 字段完整穿过 JSON 序列化");
|
||
check(
|
||
received?.metrics?.agent !== undefined && typeof received?.metrics?.agent.authGateMs === "number",
|
||
"AgentHealth 嵌套结构完整",
|
||
);
|
||
const wireBytes = JSON.stringify(hb).length;
|
||
info(`心跳报文 ${wireBytes} 字节(加指标前约 ${JSON.stringify({ ...hb, metrics: null }).length} 字节)`);
|
||
check(wireBytes < 2000, "报文仍然很小(< 2KB)");
|
||
server.stop();
|
||
|
||
// ---------------------------------------------------------------------------
|
||
head("4. 规则引擎:阈值、迟滞、冷却");
|
||
|
||
__resetHealthState();
|
||
const NOW = Date.now();
|
||
const node = (over: Partial<ClusterNode> = {}): ClusterNode => ({
|
||
id: "test-node", baseUrl: "http://x", platform: "ios", status: "online",
|
||
lastSeenAt: NOW, evalInFlight: 0, evalCapacity: 3, fixInFlight: 0, fixCapacity: 1,
|
||
version: "v", createdAt: 0, live: true, ...over,
|
||
} as ClusterNode);
|
||
const withDisk = (pct: number, over: Partial<NodeMetrics> = {}): NodeMetrics =>
|
||
({ ...readMetrics()!, at: Date.now(), diskUsedPct: pct, ...over });
|
||
|
||
noteNodeMetrics("test-node", withDisk(50));
|
||
check(judgeNode(node(), NOW).health.level === "ok", "50% 磁盘 → 正常");
|
||
|
||
noteNodeMetrics("test-node", withDisk(93));
|
||
const spike = judgeNode(node(), NOW);
|
||
check(spike.health.level === "critical", "93% 磁盘 → 面板立即标红");
|
||
check(spike.toNotify.length === 0, "但未达持续时长,不推送飞书(避免为瞬时尖峰打扰)");
|
||
|
||
noteNodeMetrics("test-node", withDisk(93));
|
||
const sustained = judgeNode(node(), NOW + 5000);
|
||
check(sustained.toNotify.length === 1, "持续超过窗口 → 推送一次");
|
||
info(`告警文案:${sustained.toNotify[0]?.message}`);
|
||
|
||
noteNodeMetrics("test-node", withDisk(93));
|
||
check(judgeNode(node(), NOW + 6000).toNotify.length === 0, "冷却期内不重复推送");
|
||
|
||
noteNodeMetrics("test-node", withDisk(87));
|
||
check(judgeNode(node(), NOW + 7000).health.level === "critical", "回落到 87%(迟滞带内)仍保持告警");
|
||
|
||
noteNodeMetrics("test-node", withDisk(84));
|
||
check(judgeNode(node(), NOW + 8000).health.level === "ok", "回落到 84%(低于清除线)→ 恢复");
|
||
|
||
__resetHealthState();
|
||
noteNodeMetrics("test-node", withDisk(50, { agent: { ...m.agent, loggedIn: false } }));
|
||
judgeNode(node(), NOW);
|
||
const agentAlert = judgeNode(node(), NOW + 5000);
|
||
check(agentAlert.toNotify.some((a) => a.rule === "agent"), "claude 未登录 → 告警");
|
||
info(`告警文案:${agentAlert.toNotify.find((a) => a.rule === "agent")?.message}`);
|
||
|
||
__resetHealthState();
|
||
noteNodeMetrics("test-node", withDisk(50, { agent: { ...m.agent, mode: "cursor", loggedIn: null } }));
|
||
judgeNode(node(), NOW);
|
||
check(
|
||
judgeNode(node(), NOW + 5000).health.alerts.every((a) => a.rule !== "agent"),
|
||
"cursor 引擎节点(loggedIn=null)默认通过,不误报",
|
||
);
|
||
|
||
__resetHealthState();
|
||
judgeNode(node({ live: false, lastSeenAt: NOW - 90_000 }), NOW);
|
||
const off = judgeNode(node({ live: false, lastSeenAt: NOW - 90_000 }), NOW + 5000);
|
||
check(off.toNotify.some((a) => a.rule === "offline"), "节点离线 → 告警");
|
||
check(off.health.alerts.length === 1, "离线时短路其余规则(不用陈旧数据判断磁盘)");
|
||
|
||
// ---------------------------------------------------------------------------
|
||
head("5. 容错:采集失败不能变成「一切正常」");
|
||
|
||
__resetHealthState();
|
||
noteNodeMetrics("test-node", withDisk(50, { diskUsedPct: null, diskFreeBytes: null }));
|
||
const unknown = judgeNode(node(), NOW);
|
||
check(unknown.health.alerts.every((a) => a.rule !== "disk"), "磁盘未知 → 不误报磁盘告警");
|
||
info("(面板此时显示「未知」而非 0%,不会被读成「空间充足」)");
|
||
|
||
__resetHealthState();
|
||
noteNodeMetrics("test-node", withDisk(50, { at: Date.now() - 600_000 }));
|
||
judgeNode(node(), NOW);
|
||
const stale = judgeNode(node(), NOW + 5000);
|
||
check(stale.health.alerts.some((a) => a.rule === "metrics_stale"), "心跳正常但采集器停摆 → 指标停更告警");
|
||
|
||
// ---------------------------------------------------------------------------
|
||
console.log(
|
||
failures === 0
|
||
? `\n\x1b[32m全部通过\x1b[0m — 采集、传输、判定、容错四段链路都验证过了。`
|
||
: `\n\x1b[31m${failures} 项失败\x1b[0m`,
|
||
);
|
||
process.exit(failures === 0 ? 0 : 1);
|