Files
2026-11-16 17:26:44 -08:00

167 lines
8.3 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { test, expect, describe } from "bun:test";
import {
DEFAULT_EVAL_PROMPT_TEMPLATE, DEFAULT_REGEN_PROMPT_TEMPLATE,
renderEvalPrompt, renderRegenPrompt, defaultTemplateFor,
} from "@bbbug/shared/eval_prompt";
import { NEEDS_INPUT_OPEN } from "@bbbug/shared/protocol";
/**
* These prompts moved out of satellite/src/evaluator.ts into a shared template so
* they could be versioned, handed down at dispatch, and replayed offline. At
* migration time the rendered output was diffed against the old inline builders
* across 24 combinations (lang × remark × kbContext × regen) and came out
* byte-identical — the English language directive was strengthened deliberately
* afterwards, and is the one intended difference.
*
* What these tests pin is the CONTRACT that made the parity possible, so a later
* edit to a template (by hand, or by an approved prompt strategy) cannot quietly
* break a downstream parser: the ASSESSMENT/<PLAN> sentinels the evaluator reads,
* the conditional KB_USED line, and the structural labels the replay scorer and
* the fix prompt locate sections by.
*/
const vars = {
lang: "zh" as const,
meegoUrl: "https://meego.example/7328764565",
title: "Story 相机特效面板打不开",
platform: "android",
bugType: "UI",
description: "点击特效按钮无反应",
reproSteps: "1. 打开 Story\n2. 点击特效",
remark: null as string | null,
kbContext: null as string | null,
};
const KB = "## 已验证的历史复现知识(参考,非权威)\n- [置信 0.87] Story 相机:入口在右上角";
describe("eval prompt rendering", () => {
test("every placeholder is substituted", () => {
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
expect(out).not.toContain("{{");
expect(out).toContain(vars.meegoUrl);
expect(out).toContain(vars.title);
});
test("the output contract the evaluator parses is present", () => {
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
expect(out).toContain("ASSESSMENT: <REPRODUCIBLE | NEEDS_INFO>");
expect(out).toContain("<PLAN>");
expect(out).toContain("</PLAN>");
expect(out).toContain(NEEDS_INPUT_OPEN);
});
test("KB_USED is asked for ONLY when something was injected", () => {
// Asking for a citation with no reference block invites invented labels, and
// an invented K2 scores a hit against a claim that was never shown.
expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars)).not.toContain("KB_USED");
const withKb = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, kbContext: KB });
expect(withKb).toContain("KB_USED");
expect(withKb).toContain("入口在右上角");
});
test("language directive follows the submitter's choice", () => {
expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars)).toStartWith("请使用中文");
const en = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, lang: "en" });
expect(en).toStartWith("OUTPUT LANGUAGE: ENGLISH");
// The directive has to survive a whole Chinese template below it.
expect(en).toContain("do not mirror their language");
// Structural labels follow the language too — they used to be frozen in
// Chinese so downstream parsers could find them, which is what produced
// English prose on a Chinese skeleton. The directive now names the English
// spelling of every label instead, and the rendered rules use it.
expect(en).toContain("复现条件 → Reproduction conditions");
expect(en).toContain("期望与实际 → Expected vs actual");
expect(en).toContain("## Expected vs actual");
expect(en).not.toContain("## 期望与实际 //");
// The directive comes FIRST, before the template's own Chinese rules.
expect(en.indexOf("OUTPUT LANGUAGE")).toBeLessThan(en.indexOf("铁律"));
});
test("a template that dropped {{langDirective}} renders no directive at all", () => {
// The guard rail for the prompt library: the directive is rendered by CODE,
// so it governs whichever version agent_prompts has active — but only while
// that version keeps the placeholder. A candidate template that drops it
// silently sends every en-selected spec back to Chinese.
const stripped = DEFAULT_EVAL_PROMPT_TEMPLATE.replace("{{langDirective}}", "");
expect(renderEvalPrompt(stripped, { ...vars, lang: "en" })).not.toContain("OUTPUT LANGUAGE");
});
test("an absent remark leaves no empty scaffold behind", () => {
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
expect(out).not.toContain("Reporter remark");
expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, remark: "需要 abid 7712334" }))
.toContain("Reporter remark / supplemental info:\n需要 abid 7712334");
});
test("empty fields degrade to (none), never to a blank line", () => {
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, description: "", reproSteps: "" });
expect(out).toContain("Description (OriginalIntro):\n(none)");
expect(out).toContain("Reproduction steps:\n(none)");
});
test("a `$&` in the ticket cannot corrupt the prompt", () => {
// Replacement-pattern syntax in user text is the classic silent corruption:
// string.replace would expand `$&` into the matched placeholder.
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, description: "价格显示 $& 和 $1" });
expect(out).toContain("价格显示 $& 和 $1");
});
test("a hand-edited template with a typo'd placeholder shows the typo", () => {
// Blanking unknown placeholders would hide the mistake inside a prompt that
// still looks plausible. Not `toBe` any more: a platform-specific device
// block is appended after the template (see the next test), so the contract
// is "the template renders unchanged at the head", not "nothing follows".
const out = renderEvalPrompt("头部 {{titel}} / {{title}}", vars);
expect(out.startsWith(`头部 {{titel}} / ${vars.title}`)).toBe(true);
expect(out).toContain("{{titel}}");
});
/**
* 评估期也要拿到设备块。
*
* 生产的 `*.eval` 模板存在 `agent_prompts` 里,没有一个版本带设备占位符 —— 只改内置
* 默认模板对生产是零效果。所以块是**渲染之后追加**的(shared/src/device_prompt.ts 的
* ensureDeviceBlockInPrompt),这几条钉的就是这条兜底路径:平台对得上才给、给了不重复、
* 别的平台一个字都不多。
*/
test("设备块按平台追加,且不重复", () => {
const android = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
expect(android).toContain("rax ttmock boe --ppe");
expect(android).toContain("enablePPE");
// 追加只发生一次,即便模板自己已经带了占位符。
const withPlaceholder = renderEvalPrompt("{{deviceBlock}}\n尾部", vars);
expect(withPlaceholder.split("rax ttmock boe --ppe").length - 1).toBe(1);
expect(withPlaceholder).toContain("尾部");
});
test("iOS 拿 catuse 那块,web 之类的平台什么都不拿", () => {
const ios = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, platform: "ios" });
expect(ios).toContain("Skill(catuse-cli)");
expect(ios).not.toContain("rax ttmock");
const other = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, platform: "web" });
expect(other).not.toContain("rax ttmock");
expect(other).not.toContain("catuse");
});
});
describe("regen prompt rendering", () => {
const ctx = { prevPlan: "上一版内容", comments: [{ quote: "复现步骤", comment: "写细一点" }] };
test("inline comments are numbered and quoted back", () => {
const out = renderRegenPrompt(DEFAULT_REGEN_PROMPT_TEMPLATE, { ...vars, ...ctx });
expect(out).toContain("1. 【原文片段】「复现步骤」");
expect(out).toContain("【人工要求】写细一点");
expect(out).toContain("上一版内容");
});
test("no comments still asks for a revision rather than a redraft", () => {
const out = renderRegenPrompt(DEFAULT_REGEN_PROMPT_TEMPLATE, { ...vars, prevPlan: "", comments: [] });
expect(out).toContain("本次没有逐条批注");
expect(out).toContain("(empty)");
});
test("defaultTemplateFor picks the right baseline per stage", () => {
expect(defaultTemplateFor("eval")).toBe(DEFAULT_EVAL_PROMPT_TEMPLATE);
expect(defaultTemplateFor("regen")).toBe(DEFAULT_REGEN_PROMPT_TEMPLATE);
});
});