Files
streamline/satellite/test/kb_injection_tracks.test.ts
T
2026-11-16 17:26:44 -08:00

98 lines
4.3 KiB
TypeScript

import { test, expect, describe } from "bun:test";
import {
DEFAULT_FEATURE_EVAL_TEMPLATE, ensureKbInPrompt, kbPromptVars,
} from "@bbbug/shared/eval_prompt";
import { buildFeatureEvalPrompt, buildFeatureRegenPrompt } from "../src/feature_evaluator";
import { buildSecurityEvalPrompt } from "../src/security_evaluator";
import type { Feature, Security } from "@bbbug/shared/types";
/**
* 三条线的评估器都得真的把知识块喂进去 —— 这是这组测试唯一想钉住的事。
*
* 背景是一次真实的空转:控制面对 bug / feature / security 三种任务都检索了知识、都往
* `kb_injections` 记了一行,但只有 bug 的评估器读了 `kbContext`。于是生产上 380 次
* feature / security 注入,提示词里一个字都没有,回报率恒为 0 —— 看板上看着像"模型
* 不采纳",实际上模型从没见过。
*
* 所以断言分两层:块在不在提示词里,以及 KB_USED 契约在不在。后者是命中率统计的唯一
* 数据来源,而且必须**只在真的注入了东西时**出现 —— 没给参考材料却要求引用编号,等于
* 邀请模型编一个 K2 出来,那比没有信号更糟。
*/
const KB = [
"## 已验证的历史复现知识(参考,非权威)",
"",
"1. 与代码调研冲突时以代码为准。",
"",
"- [K1] [置信 0.87] Story 相机:特效入口在右上角三点 → 设置",
].join("\n");
const feature = {
id: "feat-1", title: "消息盒子支持置顶", platform: "android", attempts: 1,
description: "希望能把会话置顶", remark: null, lang: "zh",
reviewHistory: [], reviewFeedback: null,
} as unknown as Feature;
const security = {
id: "sec-1", title: "剪贴板越权读取", platform: "android", attempts: 1,
reviewSpec: "检查剪贴板读取是否有前台校验", reproSpec: "冷启动后读取剪贴板",
targetRef: null, lang: "zh", reviewHistory: [], reviewFeedback: null,
} as unknown as Security;
const regenCtx = { prevPlan: "上一版方案", comments: [{ quote: "置顶", comment: "补充上限" }] };
describe("知识注入 · feature / security 评估提示词", () => {
test("feature 首轮:块进了提示词,并要求 KB_USED", () => {
const out = buildFeatureEvalPrompt(feature, null, KB);
expect(out).toContain("Story 相机:特效入口在右上角三点");
expect(out).toMatch(/^KB_USED:/m);
expect(out).not.toContain("{{");
});
test("feature 重生成:同样注入", () => {
const out = buildFeatureRegenPrompt(feature, regenCtx, null, KB);
expect(out).toContain("Story 相机:特效入口在右上角三点");
expect(out).toMatch(/^KB_USED:/m);
});
test("security:同样注入", () => {
const out = buildSecurityEvalPrompt(security, "/ws/sec-1", null, KB);
expect(out).toContain("Story 相机:特效入口在右上角三点");
expect(out).toMatch(/^KB_USED:/m);
});
test("没注入就不要 KB_USED —— 否则模型会编一个编号出来", () => {
for (const out of [
buildFeatureEvalPrompt(feature),
buildFeatureRegenPrompt(feature, regenCtx),
buildSecurityEvalPrompt(security, "/ws/sec-1"),
]) {
expect(out).not.toContain("KB_USED");
expect(out).not.toContain("{{");
}
});
});
describe("ensureKbInPrompt · 老模板的兜底", () => {
// agent_prompts 里存一份改造前拷贝来的模板,就足以让注入再次静默失效。兜底把这条
// 路堵上:模板没有占位符,渲染完照样补。
const legacy = "You are evaluating a feature.\n\nASSESSMENT: <READY | NEEDS_INFO>\nREASON: <why>\n\n<PLAN>x</PLAN>";
test("缺占位符时补上块,并把 KB_USED 插在 REASON 之后", () => {
const out = ensureKbInPrompt(legacy, KB);
expect(out).toContain("Story 相机:特效入口在右上角三点");
expect(out).toMatch(/^REASON: <why>\nKB_USED:/m);
});
test("模板已带占位符时是恒等变换", () => {
const rendered = DEFAULT_FEATURE_EVAL_TEMPLATE
.replaceAll("{{kbContext}}", kbPromptVars(KB).kbContext)
.replaceAll("{{kbUsedContract}}", kbPromptVars(KB).kbUsedContract);
expect(ensureKbInPrompt(rendered, KB)).toBe(rendered);
});
test("没有 kbContext 时什么都不做", () => {
expect(ensureKbInPrompt(legacy, null)).toBe(legacy);
expect(ensureKbInPrompt(legacy, " ")).toBe(legacy);
});
});