Initial import of streamline
This commit is contained in:
@@ -0,0 +1,203 @@
|
||||
// MUST be first: sets DATABASE_URL before db_client is evaluated.
|
||||
import "./setup_env";
|
||||
|
||||
import { test, expect, describe } from "bun:test";
|
||||
import {
|
||||
summarizeQuality, criticVerdictOf, humanRejectCount, criticBounceCount,
|
||||
type QualityInput, type QualityTaskRow,
|
||||
} from "../src/rsi_metrics";
|
||||
import { CRITIC_UNAVAILABLE } from "@bbbug/shared/types";
|
||||
|
||||
const T0 = 1_760_000_000_000;
|
||||
|
||||
function task(over: Partial<QualityTaskRow> = {}): QualityTaskRow {
|
||||
return {
|
||||
id: "t1", platform: "android", status: "fix_succeeded", attempts: 1,
|
||||
fixedRisk: 20, criticVerdict: "ROOT_CAUSE", reviewHistory: null, failureReason: null,
|
||||
mrUrl: "https://mr/1", mrMergedAt: T0, statusChangedAt: T0, finishedAt: T0,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
const history = (notes: { source: string; text: string; round?: number }[]) =>
|
||||
JSON.stringify(notes.map((n) => ({ at: T0, source: n.source, round: n.round ?? 1, text: n.text })));
|
||||
|
||||
function input(over: Partial<QualityInput> = {}): QualityInput {
|
||||
return {
|
||||
tasks: [], conditions: [], injections: [], unrepro: [],
|
||||
deliveriesByTask: new Map(),
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
const metric = (snap: ReturnType<typeof summarizeQuality>, key: string) =>
|
||||
snap.metrics.find((m) => m.key === key)!;
|
||||
|
||||
const summarize = (i: QualityInput) =>
|
||||
summarizeQuality(i, "2026-08-12", { sinceMs: T0 - 1000, untilMs: T0, days: 14 }, T0);
|
||||
|
||||
describe("critic verdict resolution", () => {
|
||||
test("prefers the column", () => {
|
||||
expect(criticVerdictOf({ criticVerdict: "SCENARIO_BANDAID" })).toBe("SCENARIO_BANDAID");
|
||||
});
|
||||
|
||||
test("falls back to the runner's own wording in failureReason", () => {
|
||||
expect(criticVerdictOf({
|
||||
criticVerdict: null,
|
||||
failureReason: "Fix-Critic 判定 INTENT_MISMATCH,已自动打回重修",
|
||||
})).toBe("INTENT_MISMATCH");
|
||||
});
|
||||
|
||||
test("falls back to the critic review note", () => {
|
||||
expect(criticVerdictOf({
|
||||
criticVerdict: null,
|
||||
reviewHistory: history([{ source: "critic", text: "【自动评审 Fix-Critic 判定:SCENARIO_BANDAID】上一批…" }]),
|
||||
})).toBe("SCENARIO_BANDAID");
|
||||
});
|
||||
|
||||
test("the latest round wins: failureReason outranks an older critic note", () => {
|
||||
expect(criticVerdictOf({
|
||||
criticVerdict: null,
|
||||
failureReason: "Fix-Critic 判定 UNCERTAIN(已达自动重修上限,转人工复核)",
|
||||
reviewHistory: history([{ source: "critic", text: "Fix-Critic 判定 SCENARIO_BANDAID" }]),
|
||||
})).toBe("UNCERTAIN");
|
||||
});
|
||||
|
||||
test("unknown / absent yields null rather than a guess", () => {
|
||||
expect(criticVerdictOf({ criticVerdict: " ", failureReason: "构建失败" })).toBeNull();
|
||||
expect(criticVerdictOf({})).toBeNull();
|
||||
});
|
||||
|
||||
test("UNAVAILABLE is ungraded, not a verdict", () => {
|
||||
expect(criticVerdictOf({ criticVerdict: CRITIC_UNAVAILABLE })).toBeNull();
|
||||
});
|
||||
|
||||
test("UNAVAILABLE does not inherit an earlier round's verdict from the lineage", () => {
|
||||
// The regression this guards: round 3 bounced as a band-aid, round 4's critic
|
||||
// crashed before judging anything. Falling through to the prose fallback would
|
||||
// hand round 4 the band-aid grade and count a process crash as 场景旁路.
|
||||
expect(criticVerdictOf({
|
||||
criticVerdict: CRITIC_UNAVAILABLE,
|
||||
failureReason: "Fix-Critic 未能运行(error_during_execution、0 turns),本批修复未经独立评审,请人工复核。",
|
||||
reviewHistory: history([{ source: "critic", text: "【自动评审 Fix-Critic 判定:SCENARIO_BANDAID】上一批…" }]),
|
||||
})).toBeNull();
|
||||
});
|
||||
|
||||
test("an ungraded round is excluded from the graded denominator", () => {
|
||||
const snap = summarize(input({
|
||||
tasks: [
|
||||
task({ id: "a", criticVerdict: "ROOT_CAUSE" }),
|
||||
task({ id: "b", criticVerdict: CRITIC_UNAVAILABLE, fixedRisk: null }),
|
||||
],
|
||||
}));
|
||||
// One task was graded, so 根因率 is 1/1 — not 1/2, which would read as if the
|
||||
// crashed run had been looked at and found wanting.
|
||||
expect(metric(snap, "rootCauseRate").denominator).toBe(1);
|
||||
expect(metric(snap, "rootCauseRate").numerator).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("review lineage counting", () => {
|
||||
test("one human note is one rejection; critic notes are counted separately", () => {
|
||||
const h = history([
|
||||
{ source: "human", text: "证据不足" },
|
||||
{ source: "critic", text: "Fix-Critic 判定 SCENARIO_BANDAID" },
|
||||
{ source: "mr_review", text: "MR 评论" },
|
||||
{ source: "human", text: "还是不行" },
|
||||
]);
|
||||
expect(humanRejectCount(h)).toBe(2);
|
||||
expect(criticBounceCount(h)).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("L0 firstPassYield", () => {
|
||||
test("approved with no rejection counts as a first pass", () => {
|
||||
const snap = summarize(input({ tasks: [task()] }));
|
||||
expect(metric(snap, "firstPassYield").value).toBe(1);
|
||||
});
|
||||
|
||||
test("approved AFTER a rejection is adjudicated but not a first pass", () => {
|
||||
const snap = summarize(input({
|
||||
tasks: [task({ reviewHistory: history([{ source: "human", text: "证据不足" }]) })],
|
||||
}));
|
||||
const m = metric(snap, "firstPassYield");
|
||||
expect(m.numerator).toBe(0);
|
||||
expect(m.denominator).toBe(1);
|
||||
});
|
||||
|
||||
test("a task still sitting in manual_review is delivered but NOT adjudicated", () => {
|
||||
// Otherwise the headline rate could be improved by simply not reviewing.
|
||||
const snap = summarize(input({
|
||||
tasks: [task({ status: "manual_review" })],
|
||||
deliveriesByTask: new Map([["t1", 1]]),
|
||||
}));
|
||||
expect(metric(snap, "firstPassYield").denominator).toBe(0);
|
||||
expect(metric(snap, "deliveries").numerator).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("L1-D risk calibration — grading the grader", () => {
|
||||
test("scores the critic's SAFE bucket against what humans actually did", () => {
|
||||
const snap = summarize(input({
|
||||
tasks: [
|
||||
task({ id: "a", fixedRisk: 10 }), // safe, passed
|
||||
task({ id: "b", fixedRisk: 20, reviewHistory: history([{ source: "human", text: "x" }]) }), // safe, rejected
|
||||
task({ id: "c", fixedRisk: 90, reviewHistory: history([{ source: "human", text: "y" }]) }), // risky, rejected
|
||||
],
|
||||
}));
|
||||
const cal = metric(snap, "riskCalibration");
|
||||
expect(cal.numerator).toBe(1);
|
||||
expect(cal.denominator).toBe(2);
|
||||
// safe 50% − risky 0% = +0.5 of discrimination.
|
||||
expect(metric(snap, "riskDiscrimination").value).toBe(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("L1-E spec prediction", () => {
|
||||
test("only verified conditions are graded — assumed rows would be self-grading", () => {
|
||||
const snap = summarize(input({
|
||||
tasks: [task()],
|
||||
conditions: [
|
||||
{ targetKind: "bug", targetId: "t1", subject: "入口", scope: "story", verdict: "verified", spec: "predicted", ts: T0 },
|
||||
{ targetKind: "bug", targetId: "t1", subject: "账号", scope: "story", verdict: "verified", spec: "contradicted", ts: T0 },
|
||||
{ targetKind: "bug", targetId: "t1", subject: "abid", scope: "story", verdict: "assumed", spec: "predicted", ts: T0 },
|
||||
],
|
||||
}));
|
||||
expect(metric(snap, "specPredictionRate").numerator).toBe(1);
|
||||
expect(metric(snap, "specPredictionRate").denominator).toBe(2);
|
||||
expect(snap.detail.topSpecMisses[0]).toEqual({ subject: "账号", scope: "story", count: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("L1-I knowledge loop", () => {
|
||||
test("un-reported injections are missing data, not zero adoption", () => {
|
||||
const snap = summarize(input({
|
||||
tasks: [task()],
|
||||
injections: [
|
||||
{ targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: null, outcome: null, ts: T0 },
|
||||
{ targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: "[]", outcome: "inconclusive", ts: T0 },
|
||||
{ targetKind: "bug", targetId: "t1", stage: "eval", usedClaimIds: '["c1"]', outcome: "confirmed", ts: T0 },
|
||||
],
|
||||
}));
|
||||
const adoption = metric(snap, "kbAdoptionRate");
|
||||
expect(adoption.denominator).toBe(2); // the null one is excluded
|
||||
expect(adoption.numerator).toBe(1);
|
||||
const hit = metric(snap, "kbHitRate");
|
||||
expect(hit.denominator).toBe(1); // only confirmed/contradicted are scored
|
||||
expect(hit.numerator).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("reliability", () => {
|
||||
test("a rate under the minimum sample is reported but flagged unreliable", () => {
|
||||
const snap = summarize(input({ tasks: [task()] }));
|
||||
expect(metric(snap, "firstPassYield").reliable).toBe(false);
|
||||
});
|
||||
|
||||
test("five adjudicated tasks clears the bar", () => {
|
||||
const snap = summarize(input({
|
||||
tasks: [1, 2, 3, 4, 5].map((i) => task({ id: `t${i}` })),
|
||||
}));
|
||||
expect(metric(snap, "firstPassYield").reliable).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user