import { test, expect, describe } from "bun:test"; import { DEFAULT_EVAL_PROMPT_TEMPLATE, DEFAULT_REGEN_PROMPT_TEMPLATE, renderEvalPrompt, renderRegenPrompt, defaultTemplateFor, } from "@bbbug/shared/eval_prompt"; import { NEEDS_INPUT_OPEN } from "@bbbug/shared/protocol"; /** * These prompts moved out of satellite/src/evaluator.ts into a shared template so * they could be versioned, handed down at dispatch, and replayed offline. At * migration time the rendered output was diffed against the old inline builders * across 24 combinations (lang × remark × kbContext × regen) and came out * byte-identical — the English language directive was strengthened deliberately * afterwards, and is the one intended difference. * * What these tests pin is the CONTRACT that made the parity possible, so a later * edit to a template (by hand, or by an approved prompt strategy) cannot quietly * break a downstream parser: the ASSESSMENT/ sentinels the evaluator reads, * the conditional KB_USED line, and the structural labels the replay scorer and * the fix prompt locate sections by. */ const vars = { lang: "zh" as const, meegoUrl: "https://meego.example/7328764565", title: "Story 相机特效面板打不开", platform: "android", bugType: "UI", description: "点击特效按钮无反应", reproSteps: "1. 打开 Story\n2. 点击特效", remark: null as string | null, kbContext: null as string | null, }; const KB = "## 已验证的历史复现知识(参考,非权威)\n- [置信 0.87] Story 相机:入口在右上角"; describe("eval prompt rendering", () => { test("every placeholder is substituted", () => { const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars); expect(out).not.toContain("{{"); expect(out).toContain(vars.meegoUrl); expect(out).toContain(vars.title); }); test("the output contract the evaluator parses is present", () => { const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars); expect(out).toContain("ASSESSMENT: "); expect(out).toContain(""); expect(out).toContain(""); expect(out).toContain(NEEDS_INPUT_OPEN); }); test("KB_USED is asked for ONLY when something was injected", () => { // Asking for a citation with no reference block invites invented labels, and // an invented K2 scores a hit against a claim that was never shown. expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars)).not.toContain("KB_USED"); const withKb = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, kbContext: KB }); expect(withKb).toContain("KB_USED"); expect(withKb).toContain("入口在右上角"); }); test("language directive follows the submitter's choice", () => { expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars)).toStartWith("请使用中文"); const en = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, lang: "en" }); expect(en).toStartWith("OUTPUT LANGUAGE: ENGLISH"); // The directive has to survive a whole Chinese template below it. expect(en).toContain("do not mirror their language"); // Structural labels follow the language too — they used to be frozen in // Chinese so downstream parsers could find them, which is what produced // English prose on a Chinese skeleton. The directive now names the English // spelling of every label instead, and the rendered rules use it. expect(en).toContain("复现条件 → Reproduction conditions"); expect(en).toContain("期望与实际 → Expected vs actual"); expect(en).toContain("## Expected vs actual"); expect(en).not.toContain("## 期望与实际 //"); // The directive comes FIRST, before the template's own Chinese rules. expect(en.indexOf("OUTPUT LANGUAGE")).toBeLessThan(en.indexOf("铁律")); }); test("a template that dropped {{langDirective}} renders no directive at all", () => { // The guard rail for the prompt library: the directive is rendered by CODE, // so it governs whichever version agent_prompts has active — but only while // that version keeps the placeholder. A candidate template that drops it // silently sends every en-selected spec back to Chinese. const stripped = DEFAULT_EVAL_PROMPT_TEMPLATE.replace("{{langDirective}}", ""); expect(renderEvalPrompt(stripped, { ...vars, lang: "en" })).not.toContain("OUTPUT LANGUAGE"); }); test("an absent remark leaves no empty scaffold behind", () => { const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars); expect(out).not.toContain("Reporter remark"); expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, remark: "需要 abid 7712334" })) .toContain("Reporter remark / supplemental info:\n需要 abid 7712334"); }); test("empty fields degrade to (none), never to a blank line", () => { const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, description: "", reproSteps: "" }); expect(out).toContain("Description (OriginalIntro):\n(none)"); expect(out).toContain("Reproduction steps:\n(none)"); }); test("a `$&` in the ticket cannot corrupt the prompt", () => { // Replacement-pattern syntax in user text is the classic silent corruption: // string.replace would expand `$&` into the matched placeholder. const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, description: "价格显示 $& 和 $1" }); expect(out).toContain("价格显示 $& 和 $1"); }); test("a hand-edited template with a typo'd placeholder shows the typo", () => { // Blanking unknown placeholders would hide the mistake inside a prompt that // still looks plausible. Not `toBe` any more: a platform-specific device // block is appended after the template (see the next test), so the contract // is "the template renders unchanged at the head", not "nothing follows". const out = renderEvalPrompt("头部 {{titel}} / {{title}}", vars); expect(out.startsWith(`头部 {{titel}} / ${vars.title}`)).toBe(true); expect(out).toContain("{{titel}}"); }); /** * 评估期也要拿到设备块。 * * 生产的 `*.eval` 模板存在 `agent_prompts` 里,没有一个版本带设备占位符 —— 只改内置 * 默认模板对生产是零效果。所以块是**渲染之后追加**的(shared/src/device_prompt.ts 的 * ensureDeviceBlockInPrompt),这几条钉的就是这条兜底路径:平台对得上才给、给了不重复、 * 别的平台一个字都不多。 */ test("设备块按平台追加,且不重复", () => { const android = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars); expect(android).toContain("rax ttmock boe --ppe"); expect(android).toContain("enablePPE"); // 追加只发生一次,即便模板自己已经带了占位符。 const withPlaceholder = renderEvalPrompt("{{deviceBlock}}\n尾部", vars); expect(withPlaceholder.split("rax ttmock boe --ppe").length - 1).toBe(1); expect(withPlaceholder).toContain("尾部"); }); test("iOS 拿 catuse 那块,web 之类的平台什么都不拿", () => { const ios = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, platform: "ios" }); expect(ios).toContain("Skill(catuse-cli)"); expect(ios).not.toContain("rax ttmock"); const other = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, platform: "web" }); expect(other).not.toContain("rax ttmock"); expect(other).not.toContain("catuse"); }); }); describe("regen prompt rendering", () => { const ctx = { prevPlan: "上一版内容", comments: [{ quote: "复现步骤", comment: "写细一点" }] }; test("inline comments are numbered and quoted back", () => { const out = renderRegenPrompt(DEFAULT_REGEN_PROMPT_TEMPLATE, { ...vars, ...ctx }); expect(out).toContain("1. 【原文片段】「复现步骤」"); expect(out).toContain("【人工要求】写细一点"); expect(out).toContain("上一版内容"); }); test("no comments still asks for a revision rather than a redraft", () => { const out = renderRegenPrompt(DEFAULT_REGEN_PROMPT_TEMPLATE, { ...vars, prevPlan: "", comments: [] }); expect(out).toContain("本次没有逐条批注"); expect(out).toContain("(empty)"); }); test("defaultTemplateFor picks the right baseline per stage", () => { expect(defaultTemplateFor("eval")).toBe(DEFAULT_EVAL_PROMPT_TEMPLATE); expect(defaultTemplateFor("regen")).toBe(DEFAULT_REGEN_PROMPT_TEMPLATE); }); });