204 lines
8.4 KiB
TypeScript
204 lines
8.4 KiB
TypeScript
// MUST be first: sets DATABASE_URL before db_client is evaluated.
|
||
import "./setup_env";
|
||
|
||
import { test, expect, describe } from "bun:test";
|
||
import {
|
||
summarizeQuality, criticVerdictOf, humanRejectCount, criticBounceCount,
|
||
type QualityInput, type QualityTaskRow,
|
||
} from "../src/rsi_metrics";
|
||
import { CRITIC_UNAVAILABLE } from "@bbbug/shared/types";
|
||
|
||
const T0 = 1_760_000_000_000;
|
||
|
||
function task(over: Partial<QualityTaskRow> = {}): QualityTaskRow {
|
||
return {
|
||
id: "t1", platform: "android", status: "fix_succeeded", attempts: 1,
|
||
fixedRisk: 20, criticVerdict: "ROOT_CAUSE", reviewHistory: null, failureReason: null,
|
||
mrUrl: "https://mr/1", mrMergedAt: T0, statusChangedAt: T0, finishedAt: T0,
|
||
...over,
|
||
};
|
||
}
|
||
|
||
const history = (notes: { source: string; text: string; round?: number }[]) =>
|
||
JSON.stringify(notes.map((n) => ({ at: T0, source: n.source, round: n.round ?? 1, text: n.text })));
|
||
|
||
function input(over: Partial<QualityInput> = {}): QualityInput {
|
||
return {
|
||
tasks: [], conditions: [], injections: [], unrepro: [],
|
||
deliveriesByTask: new Map(),
|
||
...over,
|
||
};
|
||
}
|
||
|
||
const metric = (snap: ReturnType<typeof summarizeQuality>, key: string) =>
|
||
snap.metrics.find((m) => m.key === key)!;
|
||
|
||
const summarize = (i: QualityInput) =>
|
||
summarizeQuality(i, "2026-08-12", { sinceMs: T0 - 1000, untilMs: T0, days: 14 }, T0);
|
||
|
||
describe("critic verdict resolution", () => {
|
||
test("prefers the column", () => {
|
||
expect(criticVerdictOf({ criticVerdict: "SCENARIO_BANDAID" })).toBe("SCENARIO_BANDAID");
|
||
});
|
||
|
||
test("falls back to the runner's own wording in failureReason", () => {
|
||
expect(criticVerdictOf({
|
||
criticVerdict: null,
|
||
failureReason: "Fix-Critic 判定 INTENT_MISMATCH,已自动打回重修",
|
||
})).toBe("INTENT_MISMATCH");
|
||
});
|
||
|
||
test("falls back to the critic review note", () => {
|
||
expect(criticVerdictOf({
|
||
criticVerdict: null,
|
||
reviewHistory: history([{ source: "critic", text: "【自动评审 Fix-Critic 判定:SCENARIO_BANDAID】上一批…" }]),
|
||
})).toBe("SCENARIO_BANDAID");
|
||
});
|
||
|
||
test("the latest round wins: failureReason outranks an older critic note", () => {
|
||
expect(criticVerdictOf({
|
||
criticVerdict: null,
|
||
failureReason: "Fix-Critic 判定 UNCERTAIN(已达自动重修上限,转人工复核)",
|
||
reviewHistory: history([{ source: "critic", text: "Fix-Critic 判定 SCENARIO_BANDAID" }]),
|
||
})).toBe("UNCERTAIN");
|
||
});
|
||
|
||
test("unknown / absent yields null rather than a guess", () => {
|
||
expect(criticVerdictOf({ criticVerdict: " ", failureReason: "构建失败" })).toBeNull();
|
||
expect(criticVerdictOf({})).toBeNull();
|
||
});
|
||
|
||
test("UNAVAILABLE is ungraded, not a verdict", () => {
|
||
expect(criticVerdictOf({ criticVerdict: CRITIC_UNAVAILABLE })).toBeNull();
|
||
});
|
||
|
||
test("UNAVAILABLE does not inherit an earlier round's verdict from the lineage", () => {
|
||
// The regression this guards: round 3 bounced as a band-aid, round 4's critic
|
||
// crashed before judging anything. Falling through to the prose fallback would
|
||
// hand round 4 the band-aid grade and count a process crash as 场景旁路.
|
||
expect(criticVerdictOf({
|
||
criticVerdict: CRITIC_UNAVAILABLE,
|
||
failureReason: "Fix-Critic 未能运行(error_during_execution、0 turns),本批修复未经独立评审,请人工复核。",
|
||
reviewHistory: history([{ source: "critic", text: "【自动评审 Fix-Critic 判定:SCENARIO_BANDAID】上一批…" }]),
|
||
})).toBeNull();
|
||
});
|
||
|
||
test("an ungraded round is excluded from the graded denominator", () => {
|
||
const snap = summarize(input({
|
||
tasks: [
|
||
task({ id: "a", criticVerdict: "ROOT_CAUSE" }),
|
||
task({ id: "b", criticVerdict: CRITIC_UNAVAILABLE, fixedRisk: null }),
|
||
],
|
||
}));
|
||
// One task was graded, so 根因率 is 1/1 — not 1/2, which would read as if the
|
||
// crashed run had been looked at and found wanting.
|
||
expect(metric(snap, "rootCauseRate").denominator).toBe(1);
|
||
expect(metric(snap, "rootCauseRate").numerator).toBe(1);
|
||
});
|
||
});
|
||
|
||
describe("review lineage counting", () => {
|
||
test("one human note is one rejection; critic notes are counted separately", () => {
|
||
const h = history([
|
||
{ source: "human", text: "证据不足" },
|
||
{ source: "critic", text: "Fix-Critic 判定 SCENARIO_BANDAID" },
|
||
{ source: "mr_review", text: "MR 评论" },
|
||
{ source: "human", text: "还是不行" },
|
||
]);
|
||
expect(humanRejectCount(h)).toBe(2);
|
||
expect(criticBounceCount(h)).toBe(1);
|
||
});
|
||
});
|
||
|
||
describe("L0 firstPassYield", () => {
|
||
test("approved with no rejection counts as a first pass", () => {
|
||
const snap = summarize(input({ tasks: [task()] }));
|
||
expect(metric(snap, "firstPassYield").value).toBe(1);
|
||
});
|
||
|
||
test("approved AFTER a rejection is adjudicated but not a first pass", () => {
|
||
const snap = summarize(input({
|
||
tasks: [task({ reviewHistory: history([{ source: "human", text: "证据不足" }]) })],
|
||
}));
|
||
const m = metric(snap, "firstPassYield");
|
||
expect(m.numerator).toBe(0);
|
||
expect(m.denominator).toBe(1);
|
||
});
|
||
|
||
test("a task still sitting in manual_review is delivered but NOT adjudicated", () => {
|
||
// Otherwise the headline rate could be improved by simply not reviewing.
|
||
const snap = summarize(input({
|
||
tasks: [task({ status: "manual_review" })],
|
||
deliveriesByTask: new Map([["t1", 1]]),
|
||
}));
|
||
expect(metric(snap, "firstPassYield").denominator).toBe(0);
|
||
expect(metric(snap, "deliveries").numerator).toBe(1);
|
||
});
|
||
});
|
||
|
||
describe("L1-D risk calibration — grading the grader", () => {
|
||
test("scores the critic's SAFE bucket against what humans actually did", () => {
|
||
const snap = summarize(input({
|
||
tasks: [
|
||
task({ id: "a", fixedRisk: 10 }), // safe, passed
|
||
task({ id: "b", fixedRisk: 20, reviewHistory: history([{ source: "human", text: "x" }]) }), // safe, rejected
|
||
task({ id: "c", fixedRisk: 90, reviewHistory: history([{ source: "human", text: "y" }]) }), // risky, rejected
|
||
],
|
||
}));
|
||
const cal = metric(snap, "riskCalibration");
|
||
expect(cal.numerator).toBe(1);
|
||
expect(cal.denominator).toBe(2);
|
||
// safe 50% − risky 0% = +0.5 of discrimination.
|
||
expect(metric(snap, "riskDiscrimination").value).toBe(0.5);
|
||
});
|
||
});
|
||
|
||
describe("L1-E spec prediction", () => {
|
||
test("only verified conditions are graded — assumed rows would be self-grading", () => {
|
||
const snap = summarize(input({
|
||
tasks: [task()],
|
||
conditions: [
|
||
{ targetKind: "bug", targetId: "t1", subject: "入口", scope: "story", verdict: "verified", spec: "predicted", ts: T0 },
|
||
{ targetKind: "bug", targetId: "t1", subject: "账号", scope: "story", verdict: "verified", spec: "contradicted", ts: T0 },
|
||
{ targetKind: "bug", targetId: "t1", subject: "abid", scope: "story", verdict: "assumed", spec: "predicted", ts: T0 },
|
||
],
|
||
}));
|
||
expect(metric(snap, "specPredictionRate").numerator).toBe(1);
|
||
expect(metric(snap, "specPredictionRate").denominator).toBe(2);
|
||
expect(snap.detail.topSpecMisses[0]).toEqual({ subject: "账号", scope: "story", count: 1 });
|
||
});
|
||
});
|
||
|
||
describe("L1-I knowledge loop", () => {
|
||
test("un-reported injections are missing data, not zero adoption", () => {
|
||
const snap = summarize(input({
|
||
tasks: [task()],
|
||
injections: [
|
||
{ targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: null, outcome: null, ts: T0 },
|
||
{ targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: "[]", outcome: "inconclusive", ts: T0 },
|
||
{ targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: '["c1"]', outcome: "confirmed", ts: T0 },
|
||
],
|
||
}));
|
||
const adoption = metric(snap, "kbAdoptionRate");
|
||
expect(adoption.denominator).toBe(2); // the null one is excluded
|
||
expect(adoption.numerator).toBe(1);
|
||
const hit = metric(snap, "kbHitRate");
|
||
expect(hit.denominator).toBe(1); // only confirmed/contradicted are scored
|
||
expect(hit.numerator).toBe(1);
|
||
});
|
||
});
|
||
|
||
describe("reliability", () => {
|
||
test("a rate under the minimum sample is reported but flagged unreliable", () => {
|
||
const snap = summarize(input({ tasks: [task()] }));
|
||
expect(metric(snap, "firstPassYield").reliable).toBe(false);
|
||
});
|
||
|
||
test("five adjudicated tasks clears the bar", () => {
|
||
const snap = summarize(input({
|
||
tasks: [1, 2, 3, 4, 5].map((i) => task({ id: `t${i}` })),
|
||
}));
|
||
expect(metric(snap, "firstPassYield").reliable).toBe(true);
|
||
});
|
||
});
|