Initial import of streamline

This commit is contained in:
2026-11-16 17:26:44 -08:00
commit 8e4aec1855
1221 changed files with 274256 additions and 0 deletions
+265
View File
@@ -0,0 +1,265 @@
// MUST be first: sets DATABASE_URL + thresholds before db_client is evaluated.
import "./setup_env";
import { test, expect, describe, beforeEach } from "bun:test";
import { judgeNode, noteNodeMetrics, healthFor, __resetHealthState } from "../src/node_health";
import type { ClusterNode } from "../src/types";
import type { NodeMetrics } from "@bbbug/shared/protocol";
const SUSTAIN = 120_000;
const COOLDOWN = 1_800_000;
const T0 = 1_700_000_000_000;
function metrics(over: Partial<NodeMetrics> = {}): NodeMetrics {
return {
at: T0, cores: 8, cpuPct: 10, loadPct: 20, uptimeSec: 1000,
diskUsedPct: 50, diskFreeBytes: 100e9,
memTotalBytes: 32e9, memUsedPct: 40, swapUsedBytes: 0,
slowAt: T0, slowError: null, slowMs: 5,
agent: { mode: "pty", loggedIn: true, authMethod: "claude.ai", checkedAt: T0, error: null, authGateMs: 0 },
...over,
};
}
function node(over: Partial<ClusterNode> = {}): ClusterNode {
return {
id: "fei-ios-01", baseUrl: "http://x:5180", platform: "ios", status: "online",
lastSeenAt: T0, evalInFlight: 0, evalCapacity: 3, fixInFlight: 0, fixCapacity: 1,
version: "abc", createdAt: 0, live: true,
...over,
} as ClusterNode;
}
/** Drive a node through repeated judgements at explicit times. */
function judgeAt(n: ClusterNode, m: NodeMetrics | null, now: number) {
if (m) noteNodeMetrics(n.id, { ...m, at: now });
return judgeNode(n, now);
}
beforeEach(() => __resetHealthState());
describe("sustain window", () => {
test("a brief spike is visible on the panel but never pages", () => {
const n = node();
// Crossing 90% for well under the sustain window — a build untarring into
// the workspace does exactly this. It should show, and it should be silent.
const r1 = judgeAt(n, metrics({ diskUsedPct: 93 }), T0);
expect(r1.health.level).toBe("critical"); // visible at once
expect(r1.toNotify).toHaveLength(0); // but nobody is woken
const r2 = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + 30_000);
expect(r2.toNotify).toHaveLength(0);
// Recovered before it ever earned a notification.
const r3 = judgeAt(n, metrics({ diskUsedPct: 60 }), T0 + 60_000);
expect(r3.health.level).toBe("ok");
expect(r3.toNotify).toHaveLength(0);
});
test("sustained badness fires once the window elapses", () => {
const n = node();
judgeAt(n, metrics({ diskUsedPct: 93 }), T0);
const r = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1);
expect(r.health.level).toBe("critical");
expect(r.toNotify).toHaveLength(1);
expect(r.toNotify[0].rule).toBe("disk");
expect(r.toNotify[0].message).toContain("磁盘占用");
// `since` points at when it STARTED, not when it fired — that is what makes
// 「已持续 N 分钟」 honest.
expect(r.toNotify[0].since).toBe(T0);
});
});
describe("cooldown", () => {
test("keeps showing the alert but stops re-notifying", () => {
const n = node();
judgeAt(n, metrics({ diskUsedPct: 93 }), T0);
const first = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1);
expect(first.toNotify).toHaveLength(1);
// Still broken a minute later: visible on the panel, silent in Lark.
const again = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 61_000);
expect(again.health.level).toBe("critical");
expect(again.health.alerts).toHaveLength(1);
expect(again.toNotify).toHaveLength(0);
});
test("re-notifies once the cooldown expires", () => {
const n = node();
judgeAt(n, metrics({ diskUsedPct: 93 }), T0);
judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1);
const later = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + COOLDOWN + 2);
expect(later.toNotify).toHaveLength(1);
});
});
describe("hysteresis", () => {
test("holds the alert inside the band and only clears below the low mark", () => {
const n = node();
judgeAt(n, metrics({ diskUsedPct: 93 }), T0);
expect(judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1).health.level).toBe("critical");
// 87% is under the 90% fire line but above the 85% clear line: a naive
// `bad = pct >= 90` would call this resolved and then re-fire at 90.1.
const inBand = judgeAt(n, metrics({ diskUsedPct: 87 }), T0 + SUSTAIN + 10_000);
expect(inBand.health.level).toBe("critical");
const cleared = judgeAt(n, metrics({ diskUsedPct: 84 }), T0 + SUSTAIN + 20_000);
expect(cleared.health.level).toBe("ok");
expect(cleared.health.alerts).toHaveLength(0);
});
test("oscillating across the band does not spam notifications", () => {
const n = node();
judgeAt(n, metrics({ diskUsedPct: 93 }), T0);
let notifications = judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1).toNotify.length;
// Wobble 93 → 87 → 93 → 87 well inside the cooldown.
for (let i = 1; i <= 6; i++) {
const pct = i % 2 ? 87 : 93;
notifications += judgeAt(n, metrics({ diskUsedPct: pct }), T0 + SUSTAIN + 1 + i * 30_000).toNotify.length;
}
expect(notifications).toBe(1);
});
});
describe("agent rule", () => {
test("loggedIn === null is 'no opinion', not a failure", () => {
// A cursor-engine node, or a probe that couldn't run. Alerting here would
// page someone for every node that doesn't run claude at all.
const n = node();
const a = metrics({ agent: { ...metrics().agent, mode: "cursor", loggedIn: null } });
judgeAt(n, a, T0);
const r = judgeAt(n, a, T0 + SUSTAIN + 1);
expect(r.health.alerts.filter((x) => x.rule === "agent")).toHaveLength(0);
});
test("loggedIn === false fires after the sustain window", () => {
const n = node();
const a = metrics({ agent: { ...metrics().agent, loggedIn: false } });
judgeAt(n, a, T0);
const r = judgeAt(n, a, T0 + SUSTAIN + 1);
const agent = r.health.alerts.find((x) => x.rule === "agent");
expect(agent?.severity).toBe("critical");
expect(agent?.message).toContain("Claude");
});
test("Codex node points operators at codex login", () => {
const n = node();
const a = metrics({ agent: { ...metrics().agent, mode: "codex", loggedIn: false } });
judgeAt(n, a, T0);
const r = judgeAt(n, a, T0 + SUSTAIN + 1);
expect(r.health.alerts.find((x) => x.rule === "agent")?.message).toContain("codex login");
});
test("authGateMs > 0 fires even while the local probe still says logged in", () => {
// This is the authoritative signal: a real API response produced it, so it
// catches the server-side revocation the local probe is blind to.
const n = node();
const a = metrics({ agent: { ...metrics().agent, loggedIn: true, authGateMs: 240_000 } });
judgeAt(n, a, T0);
const r = judgeAt(n, a, T0 + SUSTAIN + 1);
const agent = r.health.alerts.find((x) => x.rule === "agent");
expect(agent).toBeDefined();
expect(agent!.message).toContain("自我停单");
});
});
describe("offline rule", () => {
test("is visible at once but only pages after the sustain window", () => {
// Total lead time is 30s (control plane declares offline) + 120s (sustain)
// ≈ 2.5 min. That is deliberate: a hot-deploy restart or a brief network
// blip resolves well inside it and must not page anyone.
const off = () => node({ live: false, status: "offline", lastSeenAt: T0 - 90_000 });
const r = judgeNode(off(), T0);
expect(r.health.level).toBe("critical"); // panel shows it immediately
expect(r.toNotify).toHaveLength(0); // but Lark stays quiet
const later = judgeNode(off(), T0 + SUSTAIN + 1);
expect(later.toNotify.map((a) => a.rule)).toContain("offline");
});
test("short-circuits the other rules", () => {
// An unreachable node has no fresh metrics; evaluating disk/agent against
// its last-known values would report a reassuring nothing.
const n = node();
noteNodeMetrics(n.id, metrics({ diskUsedPct: 99 }));
const off = () => node({ live: false, lastSeenAt: T0 - 90_000 });
judgeNode(off(), T0);
const r = judgeNode(off(), T0 + SUSTAIN + 1);
expect(r.health.alerts.map((a) => a.rule)).toEqual(["offline"]);
});
test("coming back clears offline so the next outage notifies promptly", () => {
const id = "fei-ios-01";
judgeNode(node({ id, live: false, lastSeenAt: T0 - 90_000 }), T0);
const fired = judgeNode(node({ id, live: false, lastSeenAt: T0 - 90_000 }), T0 + SUSTAIN + 1);
expect(fired.toNotify).toHaveLength(1);
// Back online → alert clears.
noteNodeMetrics(id, metrics());
const back = judgeNode(node({ id, live: true }), T0 + SUSTAIN + 10_000);
expect(back.health.level).toBe("ok");
// A fresh outage restarts its own sustain window rather than inheriting the
// old cooldown.
const t2 = T0 + SUSTAIN + 100_000;
judgeNode(node({ id, live: false, lastSeenAt: t2 - 90_000 }), t2);
const second = judgeNode(node({ id, live: false, lastSeenAt: t2 - 90_000 }), t2 + SUSTAIN + 1);
expect(second.toNotify).toHaveLength(1);
});
});
describe("unmeasurable vs healthy", () => {
test("a null disk reading never counts as healthy headroom", () => {
const n = node();
const m = metrics({ diskUsedPct: null, diskFreeBytes: null });
judgeAt(n, m, T0);
const r = judgeAt(n, m, T0 + SUSTAIN + 1);
// No disk alert (we genuinely don't know)...
expect(r.health.alerts.find((a) => a.rule === "disk")).toBeUndefined();
// ...but the reading is absent rather than 0, so the panel shows 未知 and
// nobody reads it as "plenty of room".
expect(healthFor(n.id)).not.toBeNull();
});
test("a heartbeating node whose sampler stopped raises a warning", () => {
const n = node();
// Metrics timestamped well in the past = the sampler died but the process
// is alive. Without this rule that looks identical to a healthy node.
noteNodeMetrics(n.id, metrics({ at: T0 - 600_000 }));
judgeNode(n, T0);
const r = judgeNode(n, T0 + SUSTAIN + 1);
const stale = r.health.alerts.find((a) => a.rule === "metrics_stale");
expect(stale?.severity).toBe("warn");
expect(r.health.level).toBe("warn");
});
});
describe("healthFor cache", () => {
test("is populated by judging and read back without advancing state", () => {
const n = node();
expect(healthFor(n.id)).toBeNull();
judgeAt(n, metrics({ diskUsedPct: 93 }), T0);
judgeAt(n, metrics({ diskUsedPct: 93 }), T0 + SUSTAIN + 1);
expect(healthFor(n.id)!.level).toBe("critical");
// Reading repeatedly must not consume the alert or change anything —
// enrichNode does this on every 3s broadcast.
for (let i = 0; i < 5; i++) expect(healthFor(n.id)!.level).toBe("critical");
expect(healthFor(n.id)!.alerts).toHaveLength(1);
});
});
describe("noteNodeMetrics robustness", () => {
test("ignores null/undefined without throwing", () => {
expect(() => noteNodeMetrics("n", null)).not.toThrow();
expect(() => noteNodeMetrics("n", undefined)).not.toThrow();
});
test("survives a malformed payload from an unknown satellite version", () => {
// Forward compatibility runs both ways: a satellite could ship a field we
// don't know, or omit one we expect. Neither may break a heartbeat.
expect(() => noteNodeMetrics("n", { at: T0 } as unknown as NodeMetrics)).not.toThrow();
expect(() => noteNodeMetrics("n", {} as unknown as NodeMetrics)).not.toThrow();
});
});