Files
streamline/control-server/test/rsi_metrics.test.ts
T
2026-11-16 17:26:44 -08:00

204 lines
8.4 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// MUST be first: sets DATABASE_URL before db_client is evaluated.
import "./setup_env";
import { test, expect, describe } from "bun:test";
import {
summarizeQuality, criticVerdictOf, humanRejectCount, criticBounceCount,
type QualityInput, type QualityTaskRow,
} from "../src/rsi_metrics";
import { CRITIC_UNAVAILABLE } from "@bbbug/shared/types";
const T0 = 1_760_000_000_000;
function task(over: Partial<QualityTaskRow> = {}): QualityTaskRow {
return {
id: "t1", platform: "android", status: "fix_succeeded", attempts: 1,
fixedRisk: 20, criticVerdict: "ROOT_CAUSE", reviewHistory: null, failureReason: null,
mrUrl: "https://mr/1", mrMergedAt: T0, statusChangedAt: T0, finishedAt: T0,
...over,
};
}
const history = (notes: { source: string; text: string; round?: number }[]) =>
JSON.stringify(notes.map((n) => ({ at: T0, source: n.source, round: n.round ?? 1, text: n.text })));
function input(over: Partial<QualityInput> = {}): QualityInput {
return {
tasks: [], conditions: [], injections: [], unrepro: [],
deliveriesByTask: new Map(),
...over,
};
}
const metric = (snap: ReturnType<typeof summarizeQuality>, key: string) =>
snap.metrics.find((m) => m.key === key)!;
const summarize = (i: QualityInput) =>
summarizeQuality(i, "2026-08-12", { sinceMs: T0 - 1000, untilMs: T0, days: 14 }, T0);
describe("critic verdict resolution", () => {
test("prefers the column", () => {
expect(criticVerdictOf({ criticVerdict: "SCENARIO_BANDAID" })).toBe("SCENARIO_BANDAID");
});
test("falls back to the runner's own wording in failureReason", () => {
expect(criticVerdictOf({
criticVerdict: null,
failureReason: "Fix-Critic 判定 INTENT_MISMATCH,已自动打回重修",
})).toBe("INTENT_MISMATCH");
});
test("falls back to the critic review note", () => {
expect(criticVerdictOf({
criticVerdict: null,
reviewHistory: history([{ source: "critic", text: "【自动评审 Fix-Critic 判定:SCENARIO_BANDAID】上一批…" }]),
})).toBe("SCENARIO_BANDAID");
});
test("the latest round wins: failureReason outranks an older critic note", () => {
expect(criticVerdictOf({
criticVerdict: null,
failureReason: "Fix-Critic 判定 UNCERTAIN(已达自动重修上限,转人工复核)",
reviewHistory: history([{ source: "critic", text: "Fix-Critic 判定 SCENARIO_BANDAID" }]),
})).toBe("UNCERTAIN");
});
test("unknown / absent yields null rather than a guess", () => {
expect(criticVerdictOf({ criticVerdict: " ", failureReason: "构建失败" })).toBeNull();
expect(criticVerdictOf({})).toBeNull();
});
test("UNAVAILABLE is ungraded, not a verdict", () => {
expect(criticVerdictOf({ criticVerdict: CRITIC_UNAVAILABLE })).toBeNull();
});
test("UNAVAILABLE does not inherit an earlier round's verdict from the lineage", () => {
// The regression this guards: round 3 bounced as a band-aid, round 4's critic
// crashed before judging anything. Falling through to the prose fallback would
// hand round 4 the band-aid grade and count a process crash as 场景旁路.
expect(criticVerdictOf({
criticVerdict: CRITIC_UNAVAILABLE,
failureReason: "Fix-Critic 未能运行(error_during_execution、0 turns),本批修复未经独立评审,请人工复核。",
reviewHistory: history([{ source: "critic", text: "【自动评审 Fix-Critic 判定:SCENARIO_BANDAID】上一批…" }]),
})).toBeNull();
});
test("an ungraded round is excluded from the graded denominator", () => {
const snap = summarize(input({
tasks: [
task({ id: "a", criticVerdict: "ROOT_CAUSE" }),
task({ id: "b", criticVerdict: CRITIC_UNAVAILABLE, fixedRisk: null }),
],
}));
// One task was graded, so 根因率 is 1/1 — not 1/2, which would read as if the
// crashed run had been looked at and found wanting.
expect(metric(snap, "rootCauseRate").denominator).toBe(1);
expect(metric(snap, "rootCauseRate").numerator).toBe(1);
});
});
describe("review lineage counting", () => {
test("one human note is one rejection; critic notes are counted separately", () => {
const h = history([
{ source: "human", text: "证据不足" },
{ source: "critic", text: "Fix-Critic 判定 SCENARIO_BANDAID" },
{ source: "mr_review", text: "MR 评论" },
{ source: "human", text: "还是不行" },
]);
expect(humanRejectCount(h)).toBe(2);
expect(criticBounceCount(h)).toBe(1);
});
});
describe("L0 firstPassYield", () => {
test("approved with no rejection counts as a first pass", () => {
const snap = summarize(input({ tasks: [task()] }));
expect(metric(snap, "firstPassYield").value).toBe(1);
});
test("approved AFTER a rejection is adjudicated but not a first pass", () => {
const snap = summarize(input({
tasks: [task({ reviewHistory: history([{ source: "human", text: "证据不足" }]) })],
}));
const m = metric(snap, "firstPassYield");
expect(m.numerator).toBe(0);
expect(m.denominator).toBe(1);
});
test("a task still sitting in manual_review is delivered but NOT adjudicated", () => {
// Otherwise the headline rate could be improved by simply not reviewing.
const snap = summarize(input({
tasks: [task({ status: "manual_review" })],
deliveriesByTask: new Map([["t1", 1]]),
}));
expect(metric(snap, "firstPassYield").denominator).toBe(0);
expect(metric(snap, "deliveries").numerator).toBe(1);
});
});
describe("L1-D risk calibration — grading the grader", () => {
test("scores the critic's SAFE bucket against what humans actually did", () => {
const snap = summarize(input({
tasks: [
task({ id: "a", fixedRisk: 10 }), // safe, passed
task({ id: "b", fixedRisk: 20, reviewHistory: history([{ source: "human", text: "x" }]) }), // safe, rejected
task({ id: "c", fixedRisk: 90, reviewHistory: history([{ source: "human", text: "y" }]) }), // risky, rejected
],
}));
const cal = metric(snap, "riskCalibration");
expect(cal.numerator).toBe(1);
expect(cal.denominator).toBe(2);
// safe 50% − risky 0% = +0.5 of discrimination.
expect(metric(snap, "riskDiscrimination").value).toBe(0.5);
});
});
describe("L1-E spec prediction", () => {
test("only verified conditions are graded — assumed rows would be self-grading", () => {
const snap = summarize(input({
tasks: [task()],
conditions: [
{ targetKind: "bug", targetId: "t1", subject: "入口", scope: "story", verdict: "verified", spec: "predicted", ts: T0 },
{ targetKind: "bug", targetId: "t1", subject: "账号", scope: "story", verdict: "verified", spec: "contradicted", ts: T0 },
{ targetKind: "bug", targetId: "t1", subject: "abid", scope: "story", verdict: "assumed", spec: "predicted", ts: T0 },
],
}));
expect(metric(snap, "specPredictionRate").numerator).toBe(1);
expect(metric(snap, "specPredictionRate").denominator).toBe(2);
expect(snap.detail.topSpecMisses[0]).toEqual({ subject: "账号", scope: "story", count: 1 });
});
});
describe("L1-I knowledge loop", () => {
test("un-reported injections are missing data, not zero adoption", () => {
const snap = summarize(input({
tasks: [task()],
injections: [
{ targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: null, outcome: null, ts: T0 },
{ targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: "[]", outcome: "inconclusive", ts: T0 },
{ targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: '["c1"]', outcome: "confirmed", ts: T0 },
],
}));
const adoption = metric(snap, "kbAdoptionRate");
expect(adoption.denominator).toBe(2); // the null one is excluded
expect(adoption.numerator).toBe(1);
const hit = metric(snap, "kbHitRate");
expect(hit.denominator).toBe(1); // only confirmed/contradicted are scored
expect(hit.numerator).toBe(1);
});
});
describe("reliability", () => {
test("a rate under the minimum sample is reported but flagged unreliable", () => {
const snap = summarize(input({ tasks: [task()] }));
expect(metric(snap, "firstPassYield").reliable).toBe(false);
});
test("five adjudicated tasks clears the bar", () => {
const snap = summarize(input({
tasks: [1, 2, 3, 4, 5].map((i) => task({ id: `t${i}` })),
}));
expect(metric(snap, "firstPassYield").reliable).toBe(true);
});
});