Files
streamline/satellite/test/metrics.test.ts
T
2026-11-16 17:26:44 -08:00

243 lines
10 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { test, expect, describe } from "bun:test";
import { parseVmStat, parseSwapusage, runBounded } from "../src/metrics";
/**
* The sampler's contract is a negative one — it must not be able to hurt the
* heartbeat — so most of these tests assert what does NOT happen: no throw, no
* fabricated number, no permanently-stuck state. Failure injection is done with
* real broken commands rather than mocks, because the failure we actually care
* about (a host too sick to fork) is a property of Bun.spawn, not of our code.
*/
const VM_STAT_REAL = `Mach Virtual Memory Statistics: (page size of 16384 bytes)
Pages free: 72004.
Pages active: 878401.
Pages inactive: 856511.
Pages speculative: 20516.
Pages throttled: 0.
Pages wired down: 272003.
Pages purgeable: 20790.
"Translation faults": 8299175558.
Pages occupied by compressor: 697496.
`;
describe("parseVmStat", () => {
const total = 51539607552; // 48 GiB, matching the fixture's machine
test("counts active + wired + compressed, not free", () => {
const pct = parseVmStat(VM_STAT_REAL, total)!;
// (878401 + 272003 + 697496) pages × 16384 B ÷ 48 GiB ≈ 58.7%
expect(pct).toBeCloseTo(58.7, 0);
});
test("is nowhere near what os.freemem() would imply", () => {
// The whole reason this parser exists: freemem() on macOS reports almost
// everything as used because cache pages aren't 'free'. A parser that
// agreed with it would make every threshold alarm fire permanently.
expect(parseVmStat(VM_STAT_REAL, total)!).toBeLessThan(80);
});
test("honours the page size the kernel reports, not a hardcoded 4096", () => {
const as4k = parseVmStat(VM_STAT_REAL.replace("16384 bytes", "4096 bytes"), total)!;
expect(as4k).toBeCloseTo(parseVmStat(VM_STAT_REAL, total)! / 4, 0);
});
test("returns null when a field is missing rather than treating it as zero", () => {
// A silently-dropped term would UNDER-report usage on exactly the machine
// whose vm_stat format drifted — the alarm would go quiet, not loud.
const missing = VM_STAT_REAL.replace(/Pages wired down:.*\n/, "");
expect(parseVmStat(missing, total)).toBeNull();
});
test("returns null on garbage and on a zero total", () => {
expect(parseVmStat("not vm_stat output", total)).toBeNull();
expect(parseVmStat(VM_STAT_REAL, 0)).toBeNull();
});
});
describe("parseSwapusage", () => {
test("parses megabytes", () => {
const b = parseSwapusage("total = 15360.00M used = 13671.88M free = 1688.12M (encrypted)")!;
expect(b).toBe(Math.round(13671.88 * 1024 ** 2));
});
test("parses a zero-swap host", () => {
expect(parseSwapusage("total = 0.00M used = 0.00M free = 0.00M")).toBe(0);
});
test("handles other units", () => {
expect(parseSwapusage("total = 8.00G used = 2.50G free = 5.50G")).toBe(Math.round(2.5 * 1024 ** 3));
});
test("returns null on unrecognised output", () => {
expect(parseSwapusage("")).toBeNull();
expect(parseSwapusage("swap is fine, thanks")).toBeNull();
});
});
describe("runBounded failure containment", () => {
test("kills a hanging command at the deadline instead of waiting for it", async () => {
const t0 = Date.now();
const res = await runBounded(["sleep", "30"], 300);
const elapsed = Date.now() - t0;
expect(res.ok).toBe(false);
if (!res.ok) expect(res.reason).toContain("timed out");
// The point of the whole design: a wedged probe releases the sampler fast.
expect(elapsed).toBeLessThan(3000);
});
test("reports a missing binary as spawn failure, distinctly from a timeout", async () => {
// These call for different fixes (broken PATH vs an overloaded host), so
// the operator-facing reason must tell them apart.
const res = await runBounded(["/nonexistent/definitely-not-here"], 1000);
expect(res.ok).toBe(false);
if (!res.ok) {
expect(res.reason).toContain("spawn failed");
expect(res.reason).not.toContain("timed out");
}
});
test("accepts non-zero exit when stdout still carries usable output", async () => {
const res = await runBounded(["sh", "-c", "echo payload; exit 3"], 1000);
expect(res.ok).toBe(true);
if (res.ok) expect(res.text.trim()).toBe("payload");
});
test("rejects a non-zero exit that produced nothing", async () => {
const res = await runBounded(["sh", "-c", "exit 3"], 1000);
expect(res.ok).toBe(false);
});
test("never throws, whatever it is handed", async () => {
for (const cmd of [[], [""], ["sh", "-c", "kill -9 $$"]]) {
await expect(runBounded(cmd as string[], 500)).resolves.toBeDefined();
}
});
});
describe("sampler lifecycle", () => {
/** Fresh module instance so module-level cache state doesn't leak between tests. */
async function freshSampler(env: Record<string, string> = {}) {
const saved: Record<string, string | undefined> = {};
const injected = { CLAUDE_USAGE_PROBE: "0", ...env };
for (const [k, v] of Object.entries(injected)) { saved[k] = process.env[k]; process.env[k] = v; }
// Cache-bust the module registry to get virgin module state.
const mod = await import(`../src/metrics?fresh=${Math.random()}`) as typeof import("../src/metrics");
return {
mod,
restore: () => { for (const [k, v] of Object.entries(saved)) { if (v === undefined) delete process.env[k]; else process.env[k] = v; } },
};
}
test("readMetrics() returns null before the first sample and never throws", async () => {
const { mod, restore } = await freshSampler();
try {
expect(mod.readMetrics()).toBeNull();
} finally { restore(); }
});
test("first tick publishes host vitals synchronously", async () => {
const { mod, restore } = await freshSampler();
try {
mod.startMetricsSampler();
const m = mod.readMetrics();
expect(m).not.toBeNull();
// Fast tier is available immediately; cpuPct needs a second sample for a
// baseline and is legitimately null on the first.
expect(m!.cores).toBeGreaterThan(0);
expect(m!.memTotalBytes).toBeGreaterThan(0);
expect(m!.cpuPct).toBeNull();
expect(m!.at).toBeGreaterThan(0);
expect(m!.claudeUsage ?? null).toBeNull();
} finally { restore(); }
});
test("a broken agent probe degrades one field, not the snapshot", async () => {
const { mod, restore } = await freshSampler({ CLAUDE_BIN: "/nonexistent/claude", STREAMLINE_AGENT_MODE: "pty" });
try {
mod.startMetricsSampler();
await Bun.sleep(1500);
const m = mod.readMetrics()!;
// The host vitals still arrived...
expect(m.memTotalBytes).toBeGreaterThan(0);
// ...and the agent reads "unknown", never a fabricated false. Reporting
// false here would page someone about a healthy node.
expect(m.agent.loggedIn).toBeNull();
expect(m.agent.error).toContain("spawn failed");
} finally { restore(); }
});
test("an unreadable disk path yields null, not a zero that reads as 'full'", async () => {
const { mod, restore } = await freshSampler({ METRICS_DISK_PATH: "/nonexistent/path/xyz" });
try {
mod.startMetricsSampler();
const m = mod.readMetrics()!;
expect(m.diskUsedPct).toBeNull();
expect(m.diskFreeBytes).toBeNull();
// A 0% reading would silently mean "plenty of room" on a node we can no
// longer measure — the alarm would never fire.
} finally { restore(); }
});
test("slow-tier timeout keeps the previous values and says so", async () => {
const { mod, restore } = await freshSampler({ METRICS_SLOW_TIMEOUT_MS: "1", METRICS_PROBE_TIMEOUT_MS: "1" });
try {
mod.startMetricsSampler();
await Bun.sleep(800);
const m = mod.readMetrics()!;
expect(m.slowError).toBeTruthy();
// Fast tier is untouched by the slow tier's failure.
expect(m.cores).toBeGreaterThan(0);
expect(m.at).toBeGreaterThan(0);
} finally { restore(); }
});
test("ensureSamplerAlive() resurrects a sampler that stopped ticking", async () => {
// SAMPLE_MS=60ms → the 3× staleness threshold is 180ms.
const { mod, restore } = await freshSampler({ METRICS_SAMPLE_MS: "60" });
try {
mod.startMetricsSampler();
const before = mod.readMetrics()!.at;
// Simulate the failure that actually bit this codebase before: the
// interval handle is lost (a hot reload that never re-armed it), so the
// timer stops but the process stays up and looks healthy.
const key = "__streamline_metrics_sampler__";
const g = globalThis as Record<string, unknown>;
clearInterval(g[key] as ReturnType<typeof setInterval>);
await Bun.sleep(250);
expect(mod.readMetrics()!.at).toBe(before); // confirm it really is dead
mod.ensureSamplerAlive();
expect(mod.readMetrics()!.at).toBeGreaterThan(before); // revived + sampled
await Bun.sleep(200);
expect(mod.readMetrics()!.at).toBeGreaterThan(before); // and ticking again
} finally { restore(); }
});
test("ensureSamplerAlive() is a no-op while the sampler is healthy", async () => {
const { mod, restore } = await freshSampler({ METRICS_SAMPLE_MS: "60000" });
try {
mod.startMetricsSampler();
const before = mod.readMetrics()!.at;
mod.ensureSamplerAlive();
// Must not restart (and so must not reset the cpuPct baseline) just
// because the interval is long.
expect(mod.readMetrics()!.at).toBe(before);
} finally { restore(); }
});
test("readMetrics() stays cheap enough to sit in the heartbeat path", async () => {
const { mod, restore } = await freshSampler();
try {
mod.startMetricsSampler();
const t0 = performance.now();
for (let i = 0; i < 50_000; i++) mod.readMetrics();
const perCall = ((performance.now() - t0) * 1000) / 50_000;
// It is a property read; anything above a microsecond means someone
// reintroduced work into the getter.
expect(perCall).toBeLessThan(1);
} finally { restore(); }
});
});