98 lines
4.3 KiB
TypeScript
98 lines
4.3 KiB
TypeScript
import { test, expect, describe } from "bun:test";
|
|
import {
|
|
DEFAULT_FEATURE_EVAL_TEMPLATE, ensureKbInPrompt, kbPromptVars,
|
|
} from "@bbbug/shared/eval_prompt";
|
|
import { buildFeatureEvalPrompt, buildFeatureRegenPrompt } from "../src/feature_evaluator";
|
|
import { buildSecurityEvalPrompt } from "../src/security_evaluator";
|
|
import type { Feature, Security } from "@bbbug/shared/types";
|
|
|
|
/**
|
|
* 三条线的评估器都得真的把知识块喂进去 —— 这是这组测试唯一想钉住的事。
|
|
*
|
|
* 背景是一次真实的空转:控制面对 bug / feature / security 三种任务都检索了知识、都往
|
|
* `kb_injections` 记了一行,但只有 bug 的评估器读了 `kbContext`。于是生产上 380 次
|
|
* feature / security 注入,提示词里一个字都没有,回报率恒为 0 —— 看板上看着像"模型
|
|
* 不采纳",实际上模型从没见过。
|
|
*
|
|
* 所以断言分两层:块在不在提示词里,以及 KB_USED 契约在不在。后者是命中率统计的唯一
|
|
* 数据来源,而且必须**只在真的注入了东西时**出现 —— 没给参考材料却要求引用编号,等于
|
|
* 邀请模型编一个 K2 出来,那比没有信号更糟。
|
|
*/
|
|
const KB = [
|
|
"## 已验证的历史复现知识(参考,非权威)",
|
|
"",
|
|
"1. 与代码调研冲突时以代码为准。",
|
|
"",
|
|
"- [K1] [置信 0.87] Story 相机:特效入口在右上角三点 → 设置",
|
|
].join("\n");
|
|
|
|
const feature = {
|
|
id: "feat-1", title: "消息盒子支持置顶", platform: "android", attempts: 1,
|
|
description: "希望能把会话置顶", remark: null, lang: "zh",
|
|
reviewHistory: [], reviewFeedback: null,
|
|
} as unknown as Feature;
|
|
|
|
const security = {
|
|
id: "sec-1", title: "剪贴板越权读取", platform: "android", attempts: 1,
|
|
reviewSpec: "检查剪贴板读取是否有前台校验", reproSpec: "冷启动后读取剪贴板",
|
|
targetRef: null, lang: "zh", reviewHistory: [], reviewFeedback: null,
|
|
} as unknown as Security;
|
|
|
|
const regenCtx = { prevPlan: "上一版方案", comments: [{ quote: "置顶", comment: "补充上限" }] };
|
|
|
|
describe("知识注入 · feature / security 评估提示词", () => {
|
|
test("feature 首轮:块进了提示词,并要求 KB_USED", () => {
|
|
const out = buildFeatureEvalPrompt(feature, null, KB);
|
|
expect(out).toContain("Story 相机:特效入口在右上角三点");
|
|
expect(out).toMatch(/^KB_USED:/m);
|
|
expect(out).not.toContain("{{");
|
|
});
|
|
|
|
test("feature 重生成:同样注入", () => {
|
|
const out = buildFeatureRegenPrompt(feature, regenCtx, null, KB);
|
|
expect(out).toContain("Story 相机:特效入口在右上角三点");
|
|
expect(out).toMatch(/^KB_USED:/m);
|
|
});
|
|
|
|
test("security:同样注入", () => {
|
|
const out = buildSecurityEvalPrompt(security, "/ws/sec-1", null, KB);
|
|
expect(out).toContain("Story 相机:特效入口在右上角三点");
|
|
expect(out).toMatch(/^KB_USED:/m);
|
|
});
|
|
|
|
test("没注入就不要 KB_USED —— 否则模型会编一个编号出来", () => {
|
|
for (const out of [
|
|
buildFeatureEvalPrompt(feature),
|
|
buildFeatureRegenPrompt(feature, regenCtx),
|
|
buildSecurityEvalPrompt(security, "/ws/sec-1"),
|
|
]) {
|
|
expect(out).not.toContain("KB_USED");
|
|
expect(out).not.toContain("{{");
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("ensureKbInPrompt · 老模板的兜底", () => {
|
|
// agent_prompts 里存一份改造前拷贝来的模板,就足以让注入再次静默失效。兜底把这条
|
|
// 路堵上:模板没有占位符,渲染完照样补。
|
|
const legacy = "You are evaluating a feature.\n\nASSESSMENT: <READY | NEEDS_INFO>\nREASON: <why>\n\n<PLAN>x</PLAN>";
|
|
|
|
test("缺占位符时补上块,并把 KB_USED 插在 REASON 之后", () => {
|
|
const out = ensureKbInPrompt(legacy, KB);
|
|
expect(out).toContain("Story 相机:特效入口在右上角三点");
|
|
expect(out).toMatch(/^REASON: <why>\nKB_USED:/m);
|
|
});
|
|
|
|
test("模板已带占位符时是恒等变换", () => {
|
|
const rendered = DEFAULT_FEATURE_EVAL_TEMPLATE
|
|
.replaceAll("{{kbContext}}", kbPromptVars(KB).kbContext)
|
|
.replaceAll("{{kbUsedContract}}", kbPromptVars(KB).kbUsedContract);
|
|
expect(ensureKbInPrompt(rendered, KB)).toBe(rendered);
|
|
});
|
|
|
|
test("没有 kbContext 时什么都不做", () => {
|
|
expect(ensureKbInPrompt(legacy, null)).toBe(legacy);
|
|
expect(ensureKbInPrompt(legacy, " ")).toBe(legacy);
|
|
});
|
|
});
|