/** * End-to-end smoke test for 节点健康, runnable on a laptop with nothing else set up. * * bun scripts/check-node-health.ts * * What it actually exercises: * - the REAL satellite sampler (satellite/src/metrics.ts) against this host * - the REAL control-plane rule engine (control-server/src/node_health.ts) * - the wire shape between them, over real HTTP * * What it fakes: only the transport ends. A stub control plane serves the two * internal endpoints the satellite calls, and the satellite side is driven * directly rather than by booting the whole process (which would want a * workspace, adb/xcrun, and a live claim loop). * * It touches NOTHING in production: DATABASE_URL is forced at a closed local * port, and no Lark chat is configured so the alert path is a no-op. Safe to run * on a developer machine at any time. */ // Must precede any import that pulls in the DAL — imports are hoisted, so this // has to be its own statement before the rest (see control-server/test/setup_env.ts). process.env.DATABASE_URL ||= "postgres://test:test@127.0.0.1:1/streamline_test"; delete process.env.LARK_NOTIFY_CHAT_ID; // Short windows so the run demonstrates firing in seconds instead of minutes. process.env.HEALTH_SUSTAIN_MS ||= "1000"; process.env.HEALTH_METRICS_STALE_MS ||= "180000"; // Fast sampling so cpuPct has a baseline to diff against within this run — at // the production 15s cadence a 2-second script would only ever see one tick. process.env.METRICS_SAMPLE_MS ||= "700"; const { startMetricsSampler, readMetrics } = await import("../satellite/src/metrics"); const { noteNodeMetrics, judgeNode, __resetHealthState } = await import("../control-server/src/node_health"); // Relative, not "@bbbug/shared/*": that alias only resolves from inside a // workspace package, not from scripts/ (same reason push-satellites.ts does it). import type { HeartbeatPayload, NodeMetrics } from "../shared/src/protocol"; import type { ClusterNode } from "../shared/src/types"; const ok = (s: string) => console.log(` \x1b[32m✓\x1b[0m ${s}`); const info = (s: string) => console.log(` \x1b[2m${s}\x1b[0m`); const head = (s: string) => console.log(`\n\x1b[1m${s}\x1b[0m`); let failures = 0; const check = (cond: boolean, s: string) => { if (cond) ok(s); else { failures++; console.log(` \x1b[31m✗ ${s}\x1b[0m`); } }; // --------------------------------------------------------------------------- head("1. 采集器:真实采样这台机器"); startMetricsSampler(); // Give the slow tier (vm_stat / sysctl / claude auth status) a moment to land. await Bun.sleep(2000); const m = readMetrics(); if (!m) { console.error("采样器没有产出快照"); process.exit(1); } info(`CPU ${m.cpuPct ?? "—"}% 负载 ${m.loadPct}% ${m.cores} 核`); info(`内存 ${m.memUsedPct ?? "—"}% 共 ${(m.memTotalBytes / 2 ** 30).toFixed(0)}GB swap ${((m.swapUsedBytes ?? 0) / 2 ** 30).toFixed(1)}GB`); info(`磁盘 ${m.diskUsedPct ?? "—"}% 剩余 ${((m.diskFreeBytes ?? 0) / 2 ** 30).toFixed(0)}GB`); info(`Agent ${m.agent.mode} · loggedIn=${m.agent.loggedIn} · ${m.agent.authMethod ?? "—"}`); info(`慢档耗时 ${m.slowMs}ms${m.slowError ? ` · 错误 ${m.slowError}` : ""}`); check(m.diskUsedPct !== null, "磁盘已采集"); check(m.memUsedPct !== null, "内存已采集(macOS 走 vm_stat,非 os.freemem)"); check(m.memUsedPct === null || m.memUsedPct < 97, "内存读数不是 os.freemem 的假高值(应远低于 97%)"); // --------------------------------------------------------------------------- head("2. 心跳路径:必须零 I/O"); const t0 = performance.now(); for (let i = 0; i < 200_000; i++) readMetrics(); const perCall = ((performance.now() - t0) * 1000) / 200_000; info(`readMetrics() × 200k → ${perCall.toFixed(4)} µs/次`); check(perCall < 1, "心跳读快照是纯属性读取(< 1µs)"); // --------------------------------------------------------------------------- head("3. 传输:真实 HTTP,校验 wire shape"); let received: HeartbeatPayload | null = null; const server = Bun.serve({ port: 0, async fetch(req) { const url = new URL(req.url); if (url.pathname === "/api/internal/nodes/heartbeat") { received = (await req.json()) as HeartbeatPayload; return Response.json({ ok: true }); } return Response.json({ ok: true, nodeId: "test" }); }, }); const hb: HeartbeatPayload = { nodeId: "test-node", evalInFlight: 0, evalCapacity: 3, fixInFlight: 0, fixCapacity: 1, metrics: readMetrics(), pendingVersion: null, queuedDeploy: true, }; const res = await fetch(`http://127.0.0.1:${server.port}/api/internal/nodes/heartbeat`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(hb), }); check(res.ok, "心跳 POST 成功"); check(received !== null && received!.metrics !== null, "metrics 字段完整穿过 JSON 序列化"); check( received?.metrics?.agent !== undefined && typeof received?.metrics?.agent.authGateMs === "number", "AgentHealth 嵌套结构完整", ); const wireBytes = JSON.stringify(hb).length; info(`心跳报文 ${wireBytes} 字节(加指标前约 ${JSON.stringify({ ...hb, metrics: null }).length} 字节)`); check(wireBytes < 2000, "报文仍然很小(< 2KB)"); server.stop(); // --------------------------------------------------------------------------- head("4. 规则引擎:阈值、迟滞、冷却"); __resetHealthState(); const NOW = Date.now(); const node = (over: Partial = {}): ClusterNode => ({ id: "test-node", baseUrl: "http://x", platform: "ios", status: "online", lastSeenAt: NOW, evalInFlight: 0, evalCapacity: 3, fixInFlight: 0, fixCapacity: 1, version: "v", createdAt: 0, live: true, ...over, } as ClusterNode); const withDisk = (pct: number, over: Partial = {}): NodeMetrics => ({ ...readMetrics()!, at: Date.now(), diskUsedPct: pct, ...over }); noteNodeMetrics("test-node", withDisk(50)); check(judgeNode(node(), NOW).health.level === "ok", "50% 磁盘 → 正常"); noteNodeMetrics("test-node", withDisk(93)); const spike = judgeNode(node(), NOW); check(spike.health.level === "critical", "93% 磁盘 → 面板立即标红"); check(spike.toNotify.length === 0, "但未达持续时长,不推送飞书(避免为瞬时尖峰打扰)"); noteNodeMetrics("test-node", withDisk(93)); const sustained = judgeNode(node(), NOW + 5000); check(sustained.toNotify.length === 1, "持续超过窗口 → 推送一次"); info(`告警文案:${sustained.toNotify[0]?.message}`); noteNodeMetrics("test-node", withDisk(93)); check(judgeNode(node(), NOW + 6000).toNotify.length === 0, "冷却期内不重复推送"); noteNodeMetrics("test-node", withDisk(87)); check(judgeNode(node(), NOW + 7000).health.level === "critical", "回落到 87%(迟滞带内)仍保持告警"); noteNodeMetrics("test-node", withDisk(84)); check(judgeNode(node(), NOW + 8000).health.level === "ok", "回落到 84%(低于清除线)→ 恢复"); __resetHealthState(); noteNodeMetrics("test-node", withDisk(50, { agent: { ...m.agent, loggedIn: false } })); judgeNode(node(), NOW); const agentAlert = judgeNode(node(), NOW + 5000); check(agentAlert.toNotify.some((a) => a.rule === "agent"), "claude 未登录 → 告警"); info(`告警文案:${agentAlert.toNotify.find((a) => a.rule === "agent")?.message}`); __resetHealthState(); noteNodeMetrics("test-node", withDisk(50, { agent: { ...m.agent, mode: "cursor", loggedIn: null } })); judgeNode(node(), NOW); check( judgeNode(node(), NOW + 5000).health.alerts.every((a) => a.rule !== "agent"), "cursor 引擎节点(loggedIn=null)默认通过,不误报", ); __resetHealthState(); judgeNode(node({ live: false, lastSeenAt: NOW - 90_000 }), NOW); const off = judgeNode(node({ live: false, lastSeenAt: NOW - 90_000 }), NOW + 5000); check(off.toNotify.some((a) => a.rule === "offline"), "节点离线 → 告警"); check(off.health.alerts.length === 1, "离线时短路其余规则(不用陈旧数据判断磁盘)"); // --------------------------------------------------------------------------- head("5. 容错:采集失败不能变成「一切正常」"); __resetHealthState(); noteNodeMetrics("test-node", withDisk(50, { diskUsedPct: null, diskFreeBytes: null })); const unknown = judgeNode(node(), NOW); check(unknown.health.alerts.every((a) => a.rule !== "disk"), "磁盘未知 → 不误报磁盘告警"); info("(面板此时显示「未知」而非 0%,不会被读成「空间充足」)"); __resetHealthState(); noteNodeMetrics("test-node", withDisk(50, { at: Date.now() - 600_000 })); judgeNode(node(), NOW); const stale = judgeNode(node(), NOW + 5000); check(stale.health.alerts.some((a) => a.rule === "metrics_stale"), "心跳正常但采集器停摆 → 指标停更告警"); // --------------------------------------------------------------------------- console.log( failures === 0 ? `\n\x1b[32m全部通过\x1b[0m — 采集、传输、判定、容错四段链路都验证过了。` : `\n\x1b[31m${failures} 项失败\x1b[0m`, ); process.exit(failures === 0 ? 0 : 1);