// MUST be first: sets DATABASE_URL before db_client is evaluated. import "./setup_env"; import { test, expect, describe } from "bun:test"; import { summarizeQuality, criticVerdictOf, humanRejectCount, criticBounceCount, type QualityInput, type QualityTaskRow, } from "../src/rsi_metrics"; import { CRITIC_UNAVAILABLE } from "@bbbug/shared/types"; const T0 = 1_760_000_000_000; function task(over: Partial = {}): QualityTaskRow { return { id: "t1", platform: "android", status: "fix_succeeded", attempts: 1, fixedRisk: 20, criticVerdict: "ROOT_CAUSE", reviewHistory: null, failureReason: null, mrUrl: "https://mr/1", mrMergedAt: T0, statusChangedAt: T0, finishedAt: T0, ...over, }; } const history = (notes: { source: string; text: string; round?: number }[]) => JSON.stringify(notes.map((n) => ({ at: T0, source: n.source, round: n.round ?? 1, text: n.text }))); function input(over: Partial = {}): QualityInput { return { tasks: [], conditions: [], injections: [], unrepro: [], deliveriesByTask: new Map(), ...over, }; } const metric = (snap: ReturnType, key: string) => snap.metrics.find((m) => m.key === key)!; const summarize = (i: QualityInput) => summarizeQuality(i, "2026-08-12", { sinceMs: T0 - 1000, untilMs: T0, days: 14 }, T0); describe("critic verdict resolution", () => { test("prefers the column", () => { expect(criticVerdictOf({ criticVerdict: "SCENARIO_BANDAID" })).toBe("SCENARIO_BANDAID"); }); test("falls back to the runner's own wording in failureReason", () => { expect(criticVerdictOf({ criticVerdict: null, failureReason: "Fix-Critic 判定 INTENT_MISMATCH,已自动打回重修", })).toBe("INTENT_MISMATCH"); }); test("falls back to the critic review note", () => { expect(criticVerdictOf({ criticVerdict: null, reviewHistory: history([{ source: "critic", text: "【自动评审 Fix-Critic 判定:SCENARIO_BANDAID】上一批…" }]), })).toBe("SCENARIO_BANDAID"); }); test("the latest round wins: failureReason outranks an older critic note", () => { expect(criticVerdictOf({ criticVerdict: null, failureReason: "Fix-Critic 判定 UNCERTAIN(已达自动重修上限,转人工复核)", reviewHistory: history([{ source: "critic", text: "Fix-Critic 判定 SCENARIO_BANDAID" }]), })).toBe("UNCERTAIN"); }); test("unknown / absent yields null rather than a guess", () => { expect(criticVerdictOf({ criticVerdict: " ", failureReason: "构建失败" })).toBeNull(); expect(criticVerdictOf({})).toBeNull(); }); test("UNAVAILABLE is ungraded, not a verdict", () => { expect(criticVerdictOf({ criticVerdict: CRITIC_UNAVAILABLE })).toBeNull(); }); test("UNAVAILABLE does not inherit an earlier round's verdict from the lineage", () => { // The regression this guards: round 3 bounced as a band-aid, round 4's critic // crashed before judging anything. Falling through to the prose fallback would // hand round 4 the band-aid grade and count a process crash as 场景旁路. expect(criticVerdictOf({ criticVerdict: CRITIC_UNAVAILABLE, failureReason: "Fix-Critic 未能运行(error_during_execution、0 turns),本批修复未经独立评审,请人工复核。", reviewHistory: history([{ source: "critic", text: "【自动评审 Fix-Critic 判定:SCENARIO_BANDAID】上一批…" }]), })).toBeNull(); }); test("an ungraded round is excluded from the graded denominator", () => { const snap = summarize(input({ tasks: [ task({ id: "a", criticVerdict: "ROOT_CAUSE" }), task({ id: "b", criticVerdict: CRITIC_UNAVAILABLE, fixedRisk: null }), ], })); // One task was graded, so 根因率 is 1/1 — not 1/2, which would read as if the // crashed run had been looked at and found wanting. expect(metric(snap, "rootCauseRate").denominator).toBe(1); expect(metric(snap, "rootCauseRate").numerator).toBe(1); }); }); describe("review lineage counting", () => { test("one human note is one rejection; critic notes are counted separately", () => { const h = history([ { source: "human", text: "证据不足" }, { source: "critic", text: "Fix-Critic 判定 SCENARIO_BANDAID" }, { source: "mr_review", text: "MR 评论" }, { source: "human", text: "还是不行" }, ]); expect(humanRejectCount(h)).toBe(2); expect(criticBounceCount(h)).toBe(1); }); }); describe("L0 firstPassYield", () => { test("approved with no rejection counts as a first pass", () => { const snap = summarize(input({ tasks: [task()] })); expect(metric(snap, "firstPassYield").value).toBe(1); }); test("approved AFTER a rejection is adjudicated but not a first pass", () => { const snap = summarize(input({ tasks: [task({ reviewHistory: history([{ source: "human", text: "证据不足" }]) })], })); const m = metric(snap, "firstPassYield"); expect(m.numerator).toBe(0); expect(m.denominator).toBe(1); }); test("a task still sitting in manual_review is delivered but NOT adjudicated", () => { // Otherwise the headline rate could be improved by simply not reviewing. const snap = summarize(input({ tasks: [task({ status: "manual_review" })], deliveriesByTask: new Map([["t1", 1]]), })); expect(metric(snap, "firstPassYield").denominator).toBe(0); expect(metric(snap, "deliveries").numerator).toBe(1); }); }); describe("L1-D risk calibration — grading the grader", () => { test("scores the critic's SAFE bucket against what humans actually did", () => { const snap = summarize(input({ tasks: [ task({ id: "a", fixedRisk: 10 }), // safe, passed task({ id: "b", fixedRisk: 20, reviewHistory: history([{ source: "human", text: "x" }]) }), // safe, rejected task({ id: "c", fixedRisk: 90, reviewHistory: history([{ source: "human", text: "y" }]) }), // risky, rejected ], })); const cal = metric(snap, "riskCalibration"); expect(cal.numerator).toBe(1); expect(cal.denominator).toBe(2); // safe 50% − risky 0% = +0.5 of discrimination. expect(metric(snap, "riskDiscrimination").value).toBe(0.5); }); }); describe("L1-E spec prediction", () => { test("only verified conditions are graded — assumed rows would be self-grading", () => { const snap = summarize(input({ tasks: [task()], conditions: [ { targetKind: "bug", targetId: "t1", subject: "入口", scope: "story", verdict: "verified", spec: "predicted", ts: T0 }, { targetKind: "bug", targetId: "t1", subject: "账号", scope: "story", verdict: "verified", spec: "contradicted", ts: T0 }, { targetKind: "bug", targetId: "t1", subject: "abid", scope: "story", verdict: "assumed", spec: "predicted", ts: T0 }, ], })); expect(metric(snap, "specPredictionRate").numerator).toBe(1); expect(metric(snap, "specPredictionRate").denominator).toBe(2); expect(snap.detail.topSpecMisses[0]).toEqual({ subject: "账号", scope: "story", count: 1 }); }); }); describe("L1-I knowledge loop", () => { test("un-reported injections are missing data, not zero adoption", () => { const snap = summarize(input({ tasks: [task()], injections: [ { targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: null, outcome: null, ts: T0 }, { targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: "[]", outcome: "inconclusive", ts: T0 }, { targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: '["c1"]', outcome: "confirmed", ts: T0 }, ], })); const adoption = metric(snap, "kbAdoptionRate"); expect(adoption.denominator).toBe(2); // the null one is excluded expect(adoption.numerator).toBe(1); const hit = metric(snap, "kbHitRate"); expect(hit.denominator).toBe(1); // only confirmed/contradicted are scored expect(hit.numerator).toBe(1); }); }); describe("reliability", () => { test("a rate under the minimum sample is reported but flagged unreliable", () => { const snap = summarize(input({ tasks: [task()] })); expect(metric(snap, "firstPassYield").reliable).toBe(false); }); test("five adjudicated tasks clears the bar", () => { const snap = summarize(input({ tasks: [1, 2, 3, 4, 5].map((i) => task({ id: `t${i}` })), })); expect(metric(snap, "firstPassYield").reliable).toBe(true); }); });