98 lines
4.5 KiB
TypeScript
98 lines
4.5 KiB
TypeScript
import { test, expect, describe, afterEach } from "bun:test";
|
|
import { mkdtempSync, rmSync, readFileSync, existsSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { setRunLog, clearRunLog, runLogWrite, noteKill, logAgentExit } from "../src/run_log";
|
|
|
|
/**
|
|
* 这一组守的是「一批执行为什么收场」这件事在盘上留得下来。
|
|
*
|
|
* 此前卫星侧的运行事件(判定卡死、被 abort、进程退出码)只进 insertLog() 和一条
|
|
* 没人接的 WS 事件,而 insertLog 在所有联网节点上是空实现 —— 于是一次被信号打断的
|
|
* 执行,盘上只剩戛然而止的会话日志,凶手是谁只能靠形状反推。
|
|
*/
|
|
const tmpDirs: string[] = [];
|
|
function workspace(): string {
|
|
const dir = mkdtempSync(path.join(tmpdir(), "run-log-"));
|
|
tmpDirs.push(dir);
|
|
return dir;
|
|
}
|
|
afterEach(() => {
|
|
while (tmpDirs.length) rmSync(tmpDirs.pop()!, { recursive: true, force: true });
|
|
});
|
|
|
|
function lines(file: string): Record<string, unknown>[] {
|
|
return readFileSync(file, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l));
|
|
}
|
|
|
|
describe("运行事件日志的落盘", () => {
|
|
test("写在会话日志旁边,一行一个 JSON 事件", () => {
|
|
const dir = workspace();
|
|
const sessionLog = path.join(dir, "logs", "session-r2.log");
|
|
setRunLog("owner-a", sessionLog);
|
|
runLogWrite("owner-a", "warn", "runner", "日志无增长,判定为卡死");
|
|
|
|
const runnerLog = path.join(dir, "logs", "runner-r2.log");
|
|
expect(existsSync(runnerLog)).toBe(true);
|
|
const [entry] = lines(runnerLog);
|
|
// 逐行 JSON:/logs/:name 那条读取通道会把不以 { 开头的行全丢掉。
|
|
expect(entry).toMatchObject({ type: "runner", level: "warn", stage: "runner", owner: "owner-a" });
|
|
expect(entry.message).toBe("日志无增长,判定为卡死");
|
|
clearRunLog("owner-a");
|
|
});
|
|
|
|
test("追加而不是覆盖,解绑之后不再往老文件里写", () => {
|
|
const dir = workspace();
|
|
const sessionLog = path.join(dir, "logs", "session-r1.log");
|
|
setRunLog("owner-b", sessionLog);
|
|
runLogWrite("owner-b", "info", "runner", "第一条");
|
|
runLogWrite("owner-b", "info", "runner", "第二条");
|
|
clearRunLog("owner-b");
|
|
runLogWrite("owner-b", "info", "runner", "解绑之后的第三条");
|
|
|
|
expect(lines(path.join(dir, "logs", "runner-r1.log")).map((e) => e.message)).toEqual(["第一条", "第二条"]);
|
|
});
|
|
});
|
|
|
|
/** 绑一份运行事件日志,记一次进程收场,把落盘的那一行读回来。 */
|
|
function exitEntry(
|
|
owner: string, proc: { exitCode: number | null; signalCode: string | null }, elapsedMs = 1_000,
|
|
): Record<string, unknown> {
|
|
const dir = workspace();
|
|
setRunLog(owner, path.join(dir, "logs", "session-r1.log"));
|
|
logAgentExit(owner, "runner", proc, Date.now() - elapsedMs);
|
|
const entries = lines(path.join(dir, "logs", "runner-r1.log"));
|
|
clearRunLog(owner);
|
|
return entries[entries.length - 1]!;
|
|
}
|
|
|
|
describe("进程收场的取证", () => {
|
|
test("我们自己杀的:理由如实记下,且只认领一次", () => {
|
|
noteKill("owner-c", "stall-timeout");
|
|
expect(exitEntry("owner-c", { exitCode: 143, signalCode: null }, 61_000).message)
|
|
.toContain("killedBy=stall-timeout");
|
|
// 理由被取走了 —— 下一次退出不该继续顶着上一次的锅。
|
|
expect(exitEntry("owner-c", { exitCode: 0, signalCode: null }).message).toContain("killedBy=self");
|
|
});
|
|
|
|
test("没人认领的信号 = 进程外面来的,这正是要区分出来的那种死法", () => {
|
|
expect(exitEntry("owner-d", { exitCode: null, signalCode: "SIGKILL" }).message).toContain("killedBy=external");
|
|
// pty 包装器会接住 SIGTERM 自己按 143 退出,那时 signalCode 是空的。
|
|
expect(exitEntry("owner-e", { exitCode: 143, signalCode: null }).message).toContain("killedBy=external");
|
|
expect(exitEntry("owner-f", { exitCode: 137, signalCode: null }).message).toContain("killedBy=external");
|
|
});
|
|
|
|
test("正常退出既不是信号也不是我们杀的,级别也不该报警", () => {
|
|
const entry = exitEntry("owner-g", { exitCode: 0, signalCode: null }, 4_487_000);
|
|
expect(entry.level).toBe("info");
|
|
expect(entry.message).toContain("code=0");
|
|
expect(entry.message).toContain("signal=none");
|
|
expect(entry.message).toContain("killedBy=self");
|
|
expect(entry.message).toContain("用时 4487s");
|
|
});
|
|
|
|
test("非零退出记为 warn —— 面板和 grep 都靠这个级别筛", () => {
|
|
expect(exitEntry("owner-h", { exitCode: 1, signalCode: null }).level).toBe("warn");
|
|
});
|
|
});
|