Initial import of streamline

This commit is contained in:
2026-11-16 17:26:44 -08:00
commit 8e4aec1855
1221 changed files with 274256 additions and 0 deletions
+189
View File
@@ -0,0 +1,189 @@
// MUST be first: sets DATABASE_URL before db_client is evaluated.
import "./setup_env";
import { test, expect, describe } from "bun:test";
import {
classifyFailure, clusterFailures, ruleEngineFinding, type FailureSample,
} from "../src/rsi_analyst";
import {
conditionMatchesClaim, matchStrength, rollUp, MATCH_THRESHOLD, REFUTE_THRESHOLD,
} from "../src/kb_outcome";
import { diffSnapshots, scoreGap } from "../src/rsi_arbitration";
import type { QualitySnapshot } from "../src/rsi_metrics";
const T0 = 1_760_000_000_000;
// ── failure clustering ──────────────────────────────────────────────────────
describe("failure classification", () => {
test("recognises the band-aid family", () => {
expect(classifyFailure("这个修法是在该场景直接 return,把功能旁路了")).toBe("scenario_bandaid");
expect(classifyFailure("Fix-Critic 判定 SCENARIO_BANDAID")).toBe("scenario_bandaid");
});
test("evidence complaints and verification complaints are different buckets", () => {
expect(classifyFailure("截图看不出修好了什么")).toBe("weak_evidence");
expect(classifyFailure("没验证冷启动场景")).toBe("insufficient_verification");
});
test("first match wins, so one note lands in exactly one bucket", () => {
// Mentions both a band-aid and weak evidence; band-aid is the more specific
// rule and must win, or shares would sum past 100%.
expect(classifyFailure("直接 return 旁路了,而且证据也不够")).toBe("scenario_bandaid");
});
test("unmatched text is unclassified rather than forced into a bucket", () => {
expect(classifyFailure("辛苦了")).toBe("unclassified");
expect(classifyFailure("")).toBe("unclassified");
});
});
describe("clustering", () => {
const sample = (taskId: string, excerpt: string, at = T0): FailureSample =>
({ taskId, title: "t", source: "human", excerpt, at });
test("counts, shares and DISTINCT tasks", () => {
const clusters = clusterFailures([
sample("a", "旁路了"), sample("a", "又旁路了"), sample("b", "旁路"),
sample("c", "截图看不出"),
]);
const bandaid = clusters.find((c) => c.code === "scenario_bandaid")!;
expect(bandaid.count).toBe(3);
expect(bandaid.tasks).toBe(2); // one task shouting twice is not two tasks
expect(bandaid.share).toBe(0.75);
expect(clusters[0].code).toBe("scenario_bandaid"); // ranked by count
});
test("shares sum to 1 — no signal is double-counted", () => {
const clusters = clusterFailures([
sample("a", "旁路了"), sample("b", "截图看不出"), sample("c", "没验证边界"), sample("d", "辛苦了"),
]);
expect(clusters.reduce((a, c) => a + c.share, 0)).toBeCloseTo(1, 5);
});
test("the rule-engine entry states counts and never proposes anything", () => {
const f = ruleEngineFinding({
day: "2026-08-12", windowDays: 30, signals: 4,
clusters: clusterFailures([sample("a", "旁路了"), sample("b", "旁路了")]),
});
expect(f.summary).toContain("场景旁路");
expect(f.highlights.length).toBeGreaterThan(0);
});
});
// ── knowledge attribution ───────────────────────────────────────────────────
describe("injection attribution", () => {
const claim = {
scopeKey: "android:story相机", subjectKey: "入口路径",
kind: "entry_path", platform: "android",
statement: "Story 相机:特效入口在右上角三点 → 设置 → 隐私",
};
const cond = (over: Partial<{ scope: string; subject: string; kind: string; statement: string }> = {}) =>
({
scope: "Story 相机", subject: "入口路径", kind: "entry_path",
statement: "Story 相机:特效入口在右上角三点 → 设置 → 隐私", ...over,
});
test("matches on normalised subject + platform-qualified or bare scope", () => {
expect(conditionMatchesClaim(claim, cond(), "android")).toBe(true);
expect(conditionMatchesClaim(
{ ...claim, scopeKey: "story相机", platform: null }, cond({ scope: "story-相机" }), null)).toBe(true);
});
test("a different subject AND a different statement never match — a wrong refutation is the expensive error", () => {
expect(conditionMatchesClaim(
claim, cond({ subject: "实验位", statement: "需开启 abid 7712334 才能看到入口" }), "android")).toBe(false);
});
test("a restated fact matches even when the two runs named the subject differently", () => {
// The reusable tail of the base (build commands, device setup, account
// facts) recurs with the SAME statement under freshly invented subjects.
// Anchoring only on subject is how that tail went unscored.
expect(conditionMatchesClaim(
claim, cond({ subject: "特效面板可见性", scope: "故事拍摄页" }), "android")).toBe(true);
});
test("a different platform's scope does not match", () => {
expect(conditionMatchesClaim(claim, cond(), "ios")).toBe(false);
// …and still not when the platform lives only in the scope key's prefix.
expect(conditionMatchesClaim({ ...claim, platform: null }, cond(), "ios")).toBe(false);
});
test("a paraphrased subject matches — exact keys never do across runs", () => {
// The production shape: claim and condition were written months apart by
// different nodes. 49,483 real pairs produced 0 exact subject matches.
expect(conditionMatchesClaim(
{ ...claim, subjectKey: "story相机特效入口路径" },
cond({ subject: "Story 相机特效入口的路径" }),
"android",
)).toBe(true);
});
test("a different kind never matches, however similar the words", () => {
expect(conditionMatchesClaim(claim, cond({ kind: "experiment" }), "android")).toBe(false);
});
test("scope alone cannot carry a match", () => {
expect(conditionMatchesClaim(claim, cond({
subject: "冷启动耗时基线", statement: "冷启动到首帧稳定在 1.8s 上下",
}), "android")).toBe(false);
});
test("refuting takes a stronger match than confirming", () => {
// A paraphrase is good enough to confirm, not good enough to deprecate.
const s = matchStrength(
{ ...claim, subjectKey: "特效面板入口" },
cond({ subject: "特效面板的入口位置", statement: "入口在右上角三点里" }),
"android");
expect(s).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
expect(s).toBeLessThan(REFUTE_THRESHOLD);
expect(REFUTE_THRESHOLD).toBeGreaterThan(MATCH_THRESHOLD);
});
test("roll-up: any contradiction dominates, silence is inconclusive", () => {
expect(rollUp([{ claimId: "a", outcome: "confirmed" }])).toBe("confirmed");
expect(rollUp([{ claimId: "a", outcome: "contradicted" }])).toBe("contradicted");
expect(rollUp([{ claimId: "a", outcome: "confirmed" }, { claimId: "b", outcome: "contradicted" }])).toBe("mixed");
expect(rollUp([{ claimId: "a", outcome: "untouched" }])).toBe("inconclusive");
expect(rollUp([])).toBe("inconclusive");
});
});
// ── tool-gap ranking ────────────────────────────────────────────────────────
describe("tool-gap ranking", () => {
test("a flaky command outranks a reliable one at the same volume", () => {
expect(scoreGap(20, 10, 5)).toBeGreaterThan(scoreGap(20, 0, 5));
});
test("twenty runs in ONE task is damped versus twenty across many", () => {
expect(scoreGap(20, 0, 1)).toBeLessThan(scoreGap(20, 0, 10));
});
});
// ── snapshot diffing ────────────────────────────────────────────────────────
describe("snapshot deltas", () => {
const snap = (day: string, value: number | null, reliable = true): QualitySnapshot => ({
day, generatedAt: T0, window: { sinceMs: 0, untilMs: T0, days: 14 },
metrics: [{ key: "firstPassYield", label: "L0", value, numerator: 1, denominator: 10, unit: "rate", reliable }],
detail: {
byPlatform: [], criticVerdicts: [], topSpecMisses: [], unreproCategories: [],
unreproClassifierAgreement: { agreed: 0, total: 0 }, riskBuckets: [],
},
});
test("relative change, and unreliable on either side taints the delta", () => {
const d = diffSnapshots(snap("2026-08-11", 0.5), snap("2026-08-12", 0.6))[0];
expect(d.changeRate).toBeCloseTo(0.2, 5);
expect(d.reliable).toBe(true);
expect(diffSnapshots(snap("2026-08-11", 0.5, false), snap("2026-08-12", 0.6))[0].reliable).toBe(false);
});
test("no baseline yields no deltas rather than a fabricated 100%", () => {
expect(diffSnapshots(null, snap("2026-08-12", 0.6))).toEqual([]);
expect(diffSnapshots(snap("2026-08-11", 0), snap("2026-08-12", 0.6))[0].changeRate).toBeNull();
});
});