import { test, expect, describe } from "bun:test"; import { DEFAULT_FEATURE_EVAL_TEMPLATE, ensureKbInPrompt, kbPromptVars, } from "@bbbug/shared/eval_prompt"; import { buildFeatureEvalPrompt, buildFeatureRegenPrompt } from "../src/feature_evaluator"; import { buildSecurityEvalPrompt } from "../src/security_evaluator"; import type { Feature, Security } from "@bbbug/shared/types"; /** * 三条线的评估器都得真的把知识块喂进去 —— 这是这组测试唯一想钉住的事。 * * 背景是一次真实的空转:控制面对 bug / feature / security 三种任务都检索了知识、都往 * `kb_injections` 记了一行,但只有 bug 的评估器读了 `kbContext`。于是生产上 380 次 * feature / security 注入,提示词里一个字都没有,回报率恒为 0 —— 看板上看着像"模型 * 不采纳",实际上模型从没见过。 * * 所以断言分两层:块在不在提示词里,以及 KB_USED 契约在不在。后者是命中率统计的唯一 * 数据来源,而且必须**只在真的注入了东西时**出现 —— 没给参考材料却要求引用编号,等于 * 邀请模型编一个 K2 出来,那比没有信号更糟。 */ const KB = [ "## 已验证的历史复现知识(参考,非权威)", "", "1. 与代码调研冲突时以代码为准。", "", "- [K1] [置信 0.87] Story 相机:特效入口在右上角三点 → 设置", ].join("\n"); const feature = { id: "feat-1", title: "消息盒子支持置顶", platform: "android", attempts: 1, description: "希望能把会话置顶", remark: null, lang: "zh", reviewHistory: [], reviewFeedback: null, } as unknown as Feature; const security = { id: "sec-1", title: "剪贴板越权读取", platform: "android", attempts: 1, reviewSpec: "检查剪贴板读取是否有前台校验", reproSpec: "冷启动后读取剪贴板", targetRef: null, lang: "zh", reviewHistory: [], reviewFeedback: null, } as unknown as Security; const regenCtx = { prevPlan: "上一版方案", comments: [{ quote: "置顶", comment: "补充上限" }] }; describe("知识注入 · feature / security 评估提示词", () => { test("feature 首轮:块进了提示词,并要求 KB_USED", () => { const out = buildFeatureEvalPrompt(feature, null, KB); expect(out).toContain("Story 相机:特效入口在右上角三点"); expect(out).toMatch(/^KB_USED:/m); expect(out).not.toContain("{{"); }); test("feature 重生成:同样注入", () => { const out = buildFeatureRegenPrompt(feature, regenCtx, null, KB); expect(out).toContain("Story 相机:特效入口在右上角三点"); expect(out).toMatch(/^KB_USED:/m); }); test("security:同样注入", () => { const out = buildSecurityEvalPrompt(security, "/ws/sec-1", null, KB); expect(out).toContain("Story 相机:特效入口在右上角三点"); expect(out).toMatch(/^KB_USED:/m); }); test("没注入就不要 KB_USED —— 否则模型会编一个编号出来", () => { for (const out of [ buildFeatureEvalPrompt(feature), buildFeatureRegenPrompt(feature, regenCtx), buildSecurityEvalPrompt(security, "/ws/sec-1"), ]) { expect(out).not.toContain("KB_USED"); expect(out).not.toContain("{{"); } }); }); describe("ensureKbInPrompt · 老模板的兜底", () => { // agent_prompts 里存一份改造前拷贝来的模板,就足以让注入再次静默失效。兜底把这条 // 路堵上:模板没有占位符,渲染完照样补。 const legacy = "You are evaluating a feature.\n\nASSESSMENT: \nREASON: \n\nx"; test("缺占位符时补上块,并把 KB_USED 插在 REASON 之后", () => { const out = ensureKbInPrompt(legacy, KB); expect(out).toContain("Story 相机:特效入口在右上角三点"); expect(out).toMatch(/^REASON: \nKB_USED:/m); }); test("模板已带占位符时是恒等变换", () => { const rendered = DEFAULT_FEATURE_EVAL_TEMPLATE .replaceAll("{{kbContext}}", kbPromptVars(KB).kbContext) .replaceAll("{{kbUsedContract}}", kbPromptVars(KB).kbUsedContract); expect(ensureKbInPrompt(rendered, KB)).toBe(rendered); }); test("没有 kbContext 时什么都不做", () => { expect(ensureKbInPrompt(legacy, null)).toBe(legacy); expect(ensureKbInPrompt(legacy, " ")).toBe(legacy); }); });