// MUST be first: sets DATABASE_URL before db_client is evaluated. import "./setup_env"; import { test, expect, describe } from "bun:test"; import { classifyFailure, clusterFailures, ruleEngineFinding, type FailureSample, } from "../src/rsi_analyst"; import { conditionMatchesClaim, matchStrength, rollUp, MATCH_THRESHOLD, REFUTE_THRESHOLD, } from "../src/kb_outcome"; import { diffSnapshots, scoreGap } from "../src/rsi_arbitration"; import type { QualitySnapshot } from "../src/rsi_metrics"; const T0 = 1_760_000_000_000; // ── failure clustering ────────────────────────────────────────────────────── describe("failure classification", () => { test("recognises the band-aid family", () => { expect(classifyFailure("这个修法是在该场景直接 return,把功能旁路了")).toBe("scenario_bandaid"); expect(classifyFailure("Fix-Critic 判定 SCENARIO_BANDAID")).toBe("scenario_bandaid"); }); test("evidence complaints and verification complaints are different buckets", () => { expect(classifyFailure("截图看不出修好了什么")).toBe("weak_evidence"); expect(classifyFailure("没验证冷启动场景")).toBe("insufficient_verification"); }); test("first match wins, so one note lands in exactly one bucket", () => { // Mentions both a band-aid and weak evidence; band-aid is the more specific // rule and must win, or shares would sum past 100%. expect(classifyFailure("直接 return 旁路了,而且证据也不够")).toBe("scenario_bandaid"); }); test("unmatched text is unclassified rather than forced into a bucket", () => { expect(classifyFailure("辛苦了")).toBe("unclassified"); expect(classifyFailure("")).toBe("unclassified"); }); }); describe("clustering", () => { const sample = (taskId: string, excerpt: string, at = T0): FailureSample => ({ taskId, title: "t", source: "human", excerpt, at }); test("counts, shares and DISTINCT tasks", () => { const clusters = clusterFailures([ sample("a", "旁路了"), sample("a", "又旁路了"), sample("b", "旁路"), sample("c", "截图看不出"), ]); const bandaid = clusters.find((c) => c.code === "scenario_bandaid")!; expect(bandaid.count).toBe(3); expect(bandaid.tasks).toBe(2); // one task shouting twice is not two tasks expect(bandaid.share).toBe(0.75); expect(clusters[0].code).toBe("scenario_bandaid"); // ranked by count }); test("shares sum to 1 — no signal is double-counted", () => { const clusters = clusterFailures([ sample("a", "旁路了"), sample("b", "截图看不出"), sample("c", "没验证边界"), sample("d", "辛苦了"), ]); expect(clusters.reduce((a, c) => a + c.share, 0)).toBeCloseTo(1, 5); }); test("the rule-engine entry states counts and never proposes anything", () => { const f = ruleEngineFinding({ day: "2026-08-12", windowDays: 30, signals: 4, clusters: clusterFailures([sample("a", "旁路了"), sample("b", "旁路了")]), }); expect(f.summary).toContain("场景旁路"); expect(f.highlights.length).toBeGreaterThan(0); }); }); // ── knowledge attribution ─────────────────────────────────────────────────── describe("injection attribution", () => { const claim = { scopeKey: "android:story相机", subjectKey: "入口路径", kind: "entry_path", platform: "android", statement: "Story 相机:特效入口在右上角三点 → 设置 → 隐私", }; const cond = (over: Partial<{ scope: string; subject: string; kind: string; statement: string }> = {}) => ({ scope: "Story 相机", subject: "入口路径", kind: "entry_path", statement: "Story 相机:特效入口在右上角三点 → 设置 → 隐私", ...over, }); test("matches on normalised subject + platform-qualified or bare scope", () => { expect(conditionMatchesClaim(claim, cond(), "android")).toBe(true); expect(conditionMatchesClaim( { ...claim, scopeKey: "story相机", platform: null }, cond({ scope: "story-相机" }), null)).toBe(true); }); test("a different subject AND a different statement never match — a wrong refutation is the expensive error", () => { expect(conditionMatchesClaim( claim, cond({ subject: "实验位", statement: "需开启 abid 7712334 才能看到入口" }), "android")).toBe(false); }); test("a restated fact matches even when the two runs named the subject differently", () => { // The reusable tail of the base (build commands, device setup, account // facts) recurs with the SAME statement under freshly invented subjects. // Anchoring only on subject is how that tail went unscored. expect(conditionMatchesClaim( claim, cond({ subject: "特效面板可见性", scope: "故事拍摄页" }), "android")).toBe(true); }); test("a different platform's scope does not match", () => { expect(conditionMatchesClaim(claim, cond(), "ios")).toBe(false); // …and still not when the platform lives only in the scope key's prefix. expect(conditionMatchesClaim({ ...claim, platform: null }, cond(), "ios")).toBe(false); }); test("a paraphrased subject matches — exact keys never do across runs", () => { // The production shape: claim and condition were written months apart by // different nodes. 49,483 real pairs produced 0 exact subject matches. expect(conditionMatchesClaim( { ...claim, subjectKey: "story相机特效入口路径" }, cond({ subject: "Story 相机特效入口的路径" }), "android", )).toBe(true); }); test("a different kind never matches, however similar the words", () => { expect(conditionMatchesClaim(claim, cond({ kind: "experiment" }), "android")).toBe(false); }); test("scope alone cannot carry a match", () => { expect(conditionMatchesClaim(claim, cond({ subject: "冷启动耗时基线", statement: "冷启动到首帧稳定在 1.8s 上下", }), "android")).toBe(false); }); test("refuting takes a stronger match than confirming", () => { // A paraphrase is good enough to confirm, not good enough to deprecate. const s = matchStrength( { ...claim, subjectKey: "特效面板入口" }, cond({ subject: "特效面板的入口位置", statement: "入口在右上角三点里" }), "android"); expect(s).toBeGreaterThanOrEqual(MATCH_THRESHOLD); expect(s).toBeLessThan(REFUTE_THRESHOLD); expect(REFUTE_THRESHOLD).toBeGreaterThan(MATCH_THRESHOLD); }); test("roll-up: any contradiction dominates, silence is inconclusive", () => { expect(rollUp([{ claimId: "a", outcome: "confirmed" }])).toBe("confirmed"); expect(rollUp([{ claimId: "a", outcome: "contradicted" }])).toBe("contradicted"); expect(rollUp([{ claimId: "a", outcome: "confirmed" }, { claimId: "b", outcome: "contradicted" }])).toBe("mixed"); expect(rollUp([{ claimId: "a", outcome: "untouched" }])).toBe("inconclusive"); expect(rollUp([])).toBe("inconclusive"); }); }); // ── tool-gap ranking ──────────────────────────────────────────────────────── describe("tool-gap ranking", () => { test("a flaky command outranks a reliable one at the same volume", () => { expect(scoreGap(20, 10, 5)).toBeGreaterThan(scoreGap(20, 0, 5)); }); test("twenty runs in ONE task is damped versus twenty across many", () => { expect(scoreGap(20, 0, 1)).toBeLessThan(scoreGap(20, 0, 10)); }); }); // ── snapshot diffing ──────────────────────────────────────────────────────── describe("snapshot deltas", () => { const snap = (day: string, value: number | null, reliable = true): QualitySnapshot => ({ day, generatedAt: T0, window: { sinceMs: 0, untilMs: T0, days: 14 }, metrics: [{ key: "firstPassYield", label: "L0", value, numerator: 1, denominator: 10, unit: "rate", reliable }], detail: { byPlatform: [], criticVerdicts: [], topSpecMisses: [], unreproCategories: [], unreproClassifierAgreement: { agreed: 0, total: 0 }, riskBuckets: [], }, }); test("relative change, and unreliable on either side taints the delta", () => { const d = diffSnapshots(snap("2026-08-11", 0.5), snap("2026-08-12", 0.6))[0]; expect(d.changeRate).toBeCloseTo(0.2, 5); expect(d.reliable).toBe(true); expect(diffSnapshots(snap("2026-08-11", 0.5, false), snap("2026-08-12", 0.6))[0].reliable).toBe(false); }); test("no baseline yields no deltas rather than a fabricated 100%", () => { expect(diffSnapshots(null, snap("2026-08-12", 0.6))).toEqual([]); expect(diffSnapshots(snap("2026-08-11", 0), snap("2026-08-12", 0.6))[0].changeRate).toBeNull(); }); });