Initial import of streamline
This commit is contained in:
@@ -0,0 +1,189 @@
|
||||
// MUST be first: sets DATABASE_URL before db_client is evaluated.
|
||||
import "./setup_env";
|
||||
|
||||
import { test, expect, describe } from "bun:test";
|
||||
import {
|
||||
classifyFailure, clusterFailures, ruleEngineFinding, type FailureSample,
|
||||
} from "../src/rsi_analyst";
|
||||
import {
|
||||
conditionMatchesClaim, matchStrength, rollUp, MATCH_THRESHOLD, REFUTE_THRESHOLD,
|
||||
} from "../src/kb_outcome";
|
||||
|
||||
import { diffSnapshots, scoreGap } from "../src/rsi_arbitration";
|
||||
import type { QualitySnapshot } from "../src/rsi_metrics";
|
||||
|
||||
const T0 = 1_760_000_000_000;
|
||||
|
||||
// ── failure clustering ──────────────────────────────────────────────────────
|
||||
|
||||
describe("failure classification", () => {
|
||||
test("recognises the band-aid family", () => {
|
||||
expect(classifyFailure("这个修法是在该场景直接 return,把功能旁路了")).toBe("scenario_bandaid");
|
||||
expect(classifyFailure("Fix-Critic 判定 SCENARIO_BANDAID")).toBe("scenario_bandaid");
|
||||
});
|
||||
|
||||
test("evidence complaints and verification complaints are different buckets", () => {
|
||||
expect(classifyFailure("截图看不出修好了什么")).toBe("weak_evidence");
|
||||
expect(classifyFailure("没验证冷启动场景")).toBe("insufficient_verification");
|
||||
});
|
||||
|
||||
test("first match wins, so one note lands in exactly one bucket", () => {
|
||||
// Mentions both a band-aid and weak evidence; band-aid is the more specific
|
||||
// rule and must win, or shares would sum past 100%.
|
||||
expect(classifyFailure("直接 return 旁路了,而且证据也不够")).toBe("scenario_bandaid");
|
||||
});
|
||||
|
||||
test("unmatched text is unclassified rather than forced into a bucket", () => {
|
||||
expect(classifyFailure("辛苦了")).toBe("unclassified");
|
||||
expect(classifyFailure("")).toBe("unclassified");
|
||||
});
|
||||
});
|
||||
|
||||
describe("clustering", () => {
|
||||
const sample = (taskId: string, excerpt: string, at = T0): FailureSample =>
|
||||
({ taskId, title: "t", source: "human", excerpt, at });
|
||||
|
||||
test("counts, shares and DISTINCT tasks", () => {
|
||||
const clusters = clusterFailures([
|
||||
sample("a", "旁路了"), sample("a", "又旁路了"), sample("b", "旁路"),
|
||||
sample("c", "截图看不出"),
|
||||
]);
|
||||
const bandaid = clusters.find((c) => c.code === "scenario_bandaid")!;
|
||||
expect(bandaid.count).toBe(3);
|
||||
expect(bandaid.tasks).toBe(2); // one task shouting twice is not two tasks
|
||||
expect(bandaid.share).toBe(0.75);
|
||||
expect(clusters[0].code).toBe("scenario_bandaid"); // ranked by count
|
||||
});
|
||||
|
||||
test("shares sum to 1 — no signal is double-counted", () => {
|
||||
const clusters = clusterFailures([
|
||||
sample("a", "旁路了"), sample("b", "截图看不出"), sample("c", "没验证边界"), sample("d", "辛苦了"),
|
||||
]);
|
||||
expect(clusters.reduce((a, c) => a + c.share, 0)).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
test("the rule-engine entry states counts and never proposes anything", () => {
|
||||
const f = ruleEngineFinding({
|
||||
day: "2026-08-12", windowDays: 30, signals: 4,
|
||||
clusters: clusterFailures([sample("a", "旁路了"), sample("b", "旁路了")]),
|
||||
});
|
||||
expect(f.summary).toContain("场景旁路");
|
||||
expect(f.highlights.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── knowledge attribution ───────────────────────────────────────────────────
|
||||
|
||||
describe("injection attribution", () => {
|
||||
const claim = {
|
||||
scopeKey: "android:story相机", subjectKey: "入口路径",
|
||||
kind: "entry_path", platform: "android",
|
||||
statement: "Story 相机:特效入口在右上角三点 → 设置 → 隐私",
|
||||
};
|
||||
const cond = (over: Partial<{ scope: string; subject: string; kind: string; statement: string }> = {}) =>
|
||||
({
|
||||
scope: "Story 相机", subject: "入口路径", kind: "entry_path",
|
||||
statement: "Story 相机:特效入口在右上角三点 → 设置 → 隐私", ...over,
|
||||
});
|
||||
|
||||
test("matches on normalised subject + platform-qualified or bare scope", () => {
|
||||
expect(conditionMatchesClaim(claim, cond(), "android")).toBe(true);
|
||||
expect(conditionMatchesClaim(
|
||||
{ ...claim, scopeKey: "story相机", platform: null }, cond({ scope: "story-相机" }), null)).toBe(true);
|
||||
});
|
||||
|
||||
test("a different subject AND a different statement never match — a wrong refutation is the expensive error", () => {
|
||||
expect(conditionMatchesClaim(
|
||||
claim, cond({ subject: "实验位", statement: "需开启 abid 7712334 才能看到入口" }), "android")).toBe(false);
|
||||
});
|
||||
|
||||
test("a restated fact matches even when the two runs named the subject differently", () => {
|
||||
// The reusable tail of the base (build commands, device setup, account
|
||||
// facts) recurs with the SAME statement under freshly invented subjects.
|
||||
// Anchoring only on subject is how that tail went unscored.
|
||||
expect(conditionMatchesClaim(
|
||||
claim, cond({ subject: "特效面板可见性", scope: "故事拍摄页" }), "android")).toBe(true);
|
||||
});
|
||||
|
||||
test("a different platform's scope does not match", () => {
|
||||
expect(conditionMatchesClaim(claim, cond(), "ios")).toBe(false);
|
||||
// …and still not when the platform lives only in the scope key's prefix.
|
||||
expect(conditionMatchesClaim({ ...claim, platform: null }, cond(), "ios")).toBe(false);
|
||||
});
|
||||
|
||||
test("a paraphrased subject matches — exact keys never do across runs", () => {
|
||||
// The production shape: claim and condition were written months apart by
|
||||
// different nodes. 49,483 real pairs produced 0 exact subject matches.
|
||||
expect(conditionMatchesClaim(
|
||||
{ ...claim, subjectKey: "story相机特效入口路径" },
|
||||
cond({ subject: "Story 相机特效入口的路径" }),
|
||||
"android",
|
||||
)).toBe(true);
|
||||
});
|
||||
|
||||
test("a different kind never matches, however similar the words", () => {
|
||||
expect(conditionMatchesClaim(claim, cond({ kind: "experiment" }), "android")).toBe(false);
|
||||
});
|
||||
|
||||
test("scope alone cannot carry a match", () => {
|
||||
expect(conditionMatchesClaim(claim, cond({
|
||||
subject: "冷启动耗时基线", statement: "冷启动到首帧稳定在 1.8s 上下",
|
||||
}), "android")).toBe(false);
|
||||
});
|
||||
|
||||
test("refuting takes a stronger match than confirming", () => {
|
||||
// A paraphrase is good enough to confirm, not good enough to deprecate.
|
||||
const s = matchStrength(
|
||||
{ ...claim, subjectKey: "特效面板入口" },
|
||||
cond({ subject: "特效面板的入口位置", statement: "入口在右上角三点里" }),
|
||||
"android");
|
||||
expect(s).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
|
||||
expect(s).toBeLessThan(REFUTE_THRESHOLD);
|
||||
expect(REFUTE_THRESHOLD).toBeGreaterThan(MATCH_THRESHOLD);
|
||||
});
|
||||
|
||||
test("roll-up: any contradiction dominates, silence is inconclusive", () => {
|
||||
expect(rollUp([{ claimId: "a", outcome: "confirmed" }])).toBe("confirmed");
|
||||
expect(rollUp([{ claimId: "a", outcome: "contradicted" }])).toBe("contradicted");
|
||||
expect(rollUp([{ claimId: "a", outcome: "confirmed" }, { claimId: "b", outcome: "contradicted" }])).toBe("mixed");
|
||||
expect(rollUp([{ claimId: "a", outcome: "untouched" }])).toBe("inconclusive");
|
||||
expect(rollUp([])).toBe("inconclusive");
|
||||
});
|
||||
});
|
||||
|
||||
// ── tool-gap ranking ────────────────────────────────────────────────────────
|
||||
|
||||
describe("tool-gap ranking", () => {
|
||||
test("a flaky command outranks a reliable one at the same volume", () => {
|
||||
expect(scoreGap(20, 10, 5)).toBeGreaterThan(scoreGap(20, 0, 5));
|
||||
});
|
||||
|
||||
test("twenty runs in ONE task is damped versus twenty across many", () => {
|
||||
expect(scoreGap(20, 0, 1)).toBeLessThan(scoreGap(20, 0, 10));
|
||||
});
|
||||
});
|
||||
|
||||
// ── snapshot diffing ────────────────────────────────────────────────────────
|
||||
|
||||
describe("snapshot deltas", () => {
|
||||
const snap = (day: string, value: number | null, reliable = true): QualitySnapshot => ({
|
||||
day, generatedAt: T0, window: { sinceMs: 0, untilMs: T0, days: 14 },
|
||||
metrics: [{ key: "firstPassYield", label: "L0", value, numerator: 1, denominator: 10, unit: "rate", reliable }],
|
||||
detail: {
|
||||
byPlatform: [], criticVerdicts: [], topSpecMisses: [], unreproCategories: [],
|
||||
unreproClassifierAgreement: { agreed: 0, total: 0 }, riskBuckets: [],
|
||||
},
|
||||
});
|
||||
|
||||
test("relative change, and unreliable on either side taints the delta", () => {
|
||||
const d = diffSnapshots(snap("2026-08-11", 0.5), snap("2026-08-12", 0.6))[0];
|
||||
expect(d.changeRate).toBeCloseTo(0.2, 5);
|
||||
expect(d.reliable).toBe(true);
|
||||
expect(diffSnapshots(snap("2026-08-11", 0.5, false), snap("2026-08-12", 0.6))[0].reliable).toBe(false);
|
||||
});
|
||||
|
||||
test("no baseline yields no deltas rather than a fabricated 100%", () => {
|
||||
expect(diffSnapshots(null, snap("2026-08-12", 0.6))).toEqual([]);
|
||||
expect(diffSnapshots(snap("2026-08-11", 0), snap("2026-08-12", 0.6))[0].changeRate).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user