// MUST be first: sets DATABASE_URL + thresholds before db_client is evaluated. import "./setup_env"; import { test, expect, describe, beforeEach } from "bun:test"; import { judgeNode, noteNodeMetrics, healthFor, __resetHealthState } from "../src/node_health"; import type { ClusterNode } from "../src/types"; import type { NodeMetrics } from "@bbbug/shared/protocol"; const SUSTAIN = 120_000; const COOLDOWN = 1_800_000; const T0 = 1_700_000_000_000; function metrics(over: Partial = {}): NodeMetrics { return { at: T0, cores: 8, cpuPct: 10, loadPct: 20, uptimeSec: 1000, diskUsedPct: 50, diskFreeBytes: 100e9, memTotalBytes: 32e9, memUsedPct: 40, swapUsedBytes: 0, slowAt: T0, slowError: null, slowMs: 5, agent: { mode: "pty", loggedIn: true, authMethod: "claude.ai", checkedAt: T0, error: null, authGateMs: 0 }, ...over, }; } function node(over: Partial = {}): ClusterNode { return { id: "fei-ios-01", baseUrl: "http://x:5180", platform: "ios", status: "online", lastSeenAt: T0, evalInFlight: 0, evalCapacity: 3, fixInFlight: 0, fixCapacity: 1, version: "abc", createdAt: 0, live: true, ...over, } as ClusterNode; } /** Drive a node through repeated judgements at explicit times. */ function judgeAt(n: ClusterNode, m: NodeMetrics | null, now: number) { if (m) noteNodeMetrics(n.id, { ...m, at: now }); return judgeNode(n, now); } beforeEach(() => __resetHealthState()); describe("sustain window", () => { test("a brief spike is visible on the panel but never pages", () => { const n = node(); // Crossing 90% for well under the sustain window — a build untarring into // the workspace does exactly this. It should show, and it should be silent. const r1 = judgeAt(n, metrics({ diskUsedPct: 93 }), T0); expect(r1.health.level).toBe("critical"); // visible at once expect(r1.toNotify).toHaveLength(0); // but nobody is woken const r2 = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + 30_000); expect(r2.toNotify).toHaveLength(0); // Recovered before it ever earned a notification. const r3 = judgeAt(n, metrics({ diskUsedPct: 60 }), T0 + 60_000); expect(r3.health.level).toBe("ok"); expect(r3.toNotify).toHaveLength(0); }); test("sustained badness fires once the window elapses", () => { const n = node(); judgeAt(n, metrics({ diskUsedPct: 93 }), T0); const r = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1); expect(r.health.level).toBe("critical"); expect(r.toNotify).toHaveLength(1); expect(r.toNotify[0].rule).toBe("disk"); expect(r.toNotify[0].message).toContain("磁盘占用"); // `since` points at when it STARTED, not when it fired — that is what makes // 「已持续 N 分钟」 honest. expect(r.toNotify[0].since).toBe(T0); }); }); describe("cooldown", () => { test("keeps showing the alert but stops re-notifying", () => { const n = node(); judgeAt(n, metrics({ diskUsedPct: 93 }), T0); const first = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1); expect(first.toNotify).toHaveLength(1); // Still broken a minute later: visible on the panel, silent in Lark. const again = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 61_000); expect(again.health.level).toBe("critical"); expect(again.health.alerts).toHaveLength(1); expect(again.toNotify).toHaveLength(0); }); test("re-notifies once the cooldown expires", () => { const n = node(); judgeAt(n, metrics({ diskUsedPct: 93 }), T0); judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1); const later = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + COOLDOWN + 2); expect(later.toNotify).toHaveLength(1); }); }); describe("hysteresis", () => { test("holds the alert inside the band and only clears below the low mark", () => { const n = node(); judgeAt(n, metrics({ diskUsedPct: 93 }), T0); expect(judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1).health.level).toBe("critical"); // 87% is under the 90% fire line but above the 85% clear line: a naive // `bad = pct >= 90` would call this resolved and then re-fire at 90.1. const inBand = judgeAt(n, metrics({ diskUsedPct: 87 }), T0 + SUSTAIN + 10_000); expect(inBand.health.level).toBe("critical"); const cleared = judgeAt(n, metrics({ diskUsedPct: 84 }), T0 + SUSTAIN + 20_000); expect(cleared.health.level).toBe("ok"); expect(cleared.health.alerts).toHaveLength(0); }); test("oscillating across the band does not spam notifications", () => { const n = node(); judgeAt(n, metrics({ diskUsedPct: 93 }), T0); let notifications = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1).toNotify.length; // Wobble 93 → 87 → 93 → 87 well inside the cooldown. for (let i = 1; i <= 6; i++) { const pct = i % 2 ? 87 : 93; notifications += judgeAt(n, metrics({ diskUsedPct: pct }), T0 + SUSTAIN + 1 + i * 30_000).toNotify.length; } expect(notifications).toBe(1); }); }); describe("agent rule", () => { test("loggedIn === null is 'no opinion', not a failure", () => { // A cursor-engine node, or a probe that couldn't run. Alerting here would // page someone for every node that doesn't run claude at all. const n = node(); const a = metrics({ agent: { ...metrics().agent, mode: "cursor", loggedIn: null } }); judgeAt(n, a, T0); const r = judgeAt(n, a, T0 + SUSTAIN + 1); expect(r.health.alerts.filter((x) => x.rule === "agent")).toHaveLength(0); }); test("loggedIn === false fires after the sustain window", () => { const n = node(); const a = metrics({ agent: { ...metrics().agent, loggedIn: false } }); judgeAt(n, a, T0); const r = judgeAt(n, a, T0 + SUSTAIN + 1); const agent = r.health.alerts.find((x) => x.rule === "agent"); expect(agent?.severity).toBe("critical"); expect(agent?.message).toContain("Claude"); }); test("Codex node points operators at codex login", () => { const n = node(); const a = metrics({ agent: { ...metrics().agent, mode: "codex", loggedIn: false } }); judgeAt(n, a, T0); const r = judgeAt(n, a, T0 + SUSTAIN + 1); expect(r.health.alerts.find((x) => x.rule === "agent")?.message).toContain("codex login"); }); test("authGateMs > 0 fires even while the local probe still says logged in", () => { // This is the authoritative signal: a real API response produced it, so it // catches the server-side revocation the local probe is blind to. const n = node(); const a = metrics({ agent: { ...metrics().agent, loggedIn: true, authGateMs: 240_000 } }); judgeAt(n, a, T0); const r = judgeAt(n, a, T0 + SUSTAIN + 1); const agent = r.health.alerts.find((x) => x.rule === "agent"); expect(agent).toBeDefined(); expect(agent!.message).toContain("自我停单"); }); }); describe("offline rule", () => { test("is visible at once but only pages after the sustain window", () => { // Total lead time is 30s (control plane declares offline) + 120s (sustain) // ≈ 2.5 min. That is deliberate: a hot-deploy restart or a brief network // blip resolves well inside it and must not page anyone. const off = () => node({ live: false, status: "offline", lastSeenAt: T0 - 90_000 }); const r = judgeNode(off(), T0); expect(r.health.level).toBe("critical"); // panel shows it immediately expect(r.toNotify).toHaveLength(0); // but Lark stays quiet const later = judgeNode(off(), T0 + SUSTAIN + 1); expect(later.toNotify.map((a) => a.rule)).toContain("offline"); }); test("short-circuits the other rules", () => { // An unreachable node has no fresh metrics; evaluating disk/agent against // its last-known values would report a reassuring nothing. const n = node(); noteNodeMetrics(n.id, metrics({ diskUsedPct: 99 })); const off = () => node({ live: false, lastSeenAt: T0 - 90_000 }); judgeNode(off(), T0); const r = judgeNode(off(), T0 + SUSTAIN + 1); expect(r.health.alerts.map((a) => a.rule)).toEqual(["offline"]); }); test("coming back clears offline so the next outage notifies promptly", () => { const id = "fei-ios-01"; judgeNode(node({ id, live: false, lastSeenAt: T0 - 90_000 }), T0); const fired = judgeNode(node({ id, live: false, lastSeenAt: T0 - 90_000 }), T0 + SUSTAIN + 1); expect(fired.toNotify).toHaveLength(1); // Back online → alert clears. noteNodeMetrics(id, metrics()); const back = judgeNode(node({ id, live: true }), T0 + SUSTAIN + 10_000); expect(back.health.level).toBe("ok"); // A fresh outage restarts its own sustain window rather than inheriting the // old cooldown. const t2 = T0 + SUSTAIN + 100_000; judgeNode(node({ id, live: false, lastSeenAt: t2 - 90_000 }), t2); const second = judgeNode(node({ id, live: false, lastSeenAt: t2 - 90_000 }), t2 + SUSTAIN + 1); expect(second.toNotify).toHaveLength(1); }); }); describe("unmeasurable vs healthy", () => { test("a null disk reading never counts as healthy headroom", () => { const n = node(); const m = metrics({ diskUsedPct: null, diskFreeBytes: null }); judgeAt(n, m, T0); const r = judgeAt(n, m, T0 + SUSTAIN + 1); // No disk alert (we genuinely don't know)... expect(r.health.alerts.find((a) => a.rule === "disk")).toBeUndefined(); // ...but the reading is absent rather than 0, so the panel shows 未知 and // nobody reads it as "plenty of room". expect(healthFor(n.id)).not.toBeNull(); }); test("a heartbeating node whose sampler stopped raises a warning", () => { const n = node(); // Metrics timestamped well in the past = the sampler died but the process // is alive. Without this rule that looks identical to a healthy node. noteNodeMetrics(n.id, metrics({ at: T0 - 600_000 })); judgeNode(n, T0); const r = judgeNode(n, T0 + SUSTAIN + 1); const stale = r.health.alerts.find((a) => a.rule === "metrics_stale"); expect(stale?.severity).toBe("warn"); expect(r.health.level).toBe("warn"); }); }); describe("healthFor cache", () => { test("is populated by judging and read back without advancing state", () => { const n = node(); expect(healthFor(n.id)).toBeNull(); judgeAt(n, metrics({ diskUsedPct: 93 }), T0); judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1); expect(healthFor(n.id)!.level).toBe("critical"); // Reading repeatedly must not consume the alert or change anything — // enrichNode does this on every 3s broadcast. for (let i = 0; i < 5; i++) expect(healthFor(n.id)!.level).toBe("critical"); expect(healthFor(n.id)!.alerts).toHaveLength(1); }); }); describe("noteNodeMetrics robustness", () => { test("ignores null/undefined without throwing", () => { expect(() => noteNodeMetrics("n", null)).not.toThrow(); expect(() => noteNodeMetrics("n", undefined)).not.toThrow(); }); test("survives a malformed payload from an unknown satellite version", () => { // Forward compatibility runs both ways: a satellite could ship a field we // don't know, or omit one we expect. Neither may break a heartbeat. expect(() => noteNodeMetrics("n", { at: T0 } as unknown as NodeMetrics)).not.toThrow(); expect(() => noteNodeMetrics("n", {} as unknown as NodeMetrics)).not.toThrow(); }); });