Initial import of streamline
This commit is contained in:
@@ -0,0 +1,166 @@
|
||||
import { test, expect, describe } from "bun:test";
|
||||
import {
|
||||
DEFAULT_EVAL_PROMPT_TEMPLATE, DEFAULT_REGEN_PROMPT_TEMPLATE,
|
||||
renderEvalPrompt, renderRegenPrompt, defaultTemplateFor,
|
||||
} from "@bbbug/shared/eval_prompt";
|
||||
import { NEEDS_INPUT_OPEN } from "@bbbug/shared/protocol";
|
||||
|
||||
/**
|
||||
* These prompts moved out of satellite/src/evaluator.ts into a shared template so
|
||||
* they could be versioned, handed down at dispatch, and replayed offline. At
|
||||
* migration time the rendered output was diffed against the old inline builders
|
||||
* across 24 combinations (lang × remark × kbContext × regen) and came out
|
||||
* byte-identical — the English language directive was strengthened deliberately
|
||||
* afterwards, and is the one intended difference.
|
||||
*
|
||||
* What these tests pin is the CONTRACT that made the parity possible, so a later
|
||||
* edit to a template (by hand, or by an approved prompt strategy) cannot quietly
|
||||
* break a downstream parser: the ASSESSMENT/<PLAN> sentinels the evaluator reads,
|
||||
* the conditional KB_USED line, and the structural labels the replay scorer and
|
||||
* the fix prompt locate sections by.
|
||||
*/
|
||||
const vars = {
|
||||
lang: "zh" as const,
|
||||
meegoUrl: "https://meego.example/7328764565",
|
||||
title: "Story 相机特效面板打不开",
|
||||
platform: "android",
|
||||
bugType: "UI",
|
||||
description: "点击特效按钮无反应",
|
||||
reproSteps: "1. 打开 Story\n2. 点击特效",
|
||||
remark: null as string | null,
|
||||
kbContext: null as string | null,
|
||||
};
|
||||
|
||||
const KB = "## 已验证的历史复现知识(参考,非权威)\n- [置信 0.87] Story 相机:入口在右上角";
|
||||
|
||||
describe("eval prompt rendering", () => {
|
||||
test("every placeholder is substituted", () => {
|
||||
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
|
||||
expect(out).not.toContain("{{");
|
||||
expect(out).toContain(vars.meegoUrl);
|
||||
expect(out).toContain(vars.title);
|
||||
});
|
||||
|
||||
test("the output contract the evaluator parses is present", () => {
|
||||
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
|
||||
expect(out).toContain("ASSESSMENT: <REPRODUCIBLE | NEEDS_INFO>");
|
||||
expect(out).toContain("<PLAN>");
|
||||
expect(out).toContain("</PLAN>");
|
||||
expect(out).toContain(NEEDS_INPUT_OPEN);
|
||||
});
|
||||
|
||||
test("KB_USED is asked for ONLY when something was injected", () => {
|
||||
// Asking for a citation with no reference block invites invented labels, and
|
||||
// an invented K2 scores a hit against a claim that was never shown.
|
||||
expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars)).not.toContain("KB_USED");
|
||||
const withKb = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, kbContext: KB });
|
||||
expect(withKb).toContain("KB_USED");
|
||||
expect(withKb).toContain("入口在右上角");
|
||||
});
|
||||
|
||||
test("language directive follows the submitter's choice", () => {
|
||||
expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars)).toStartWith("请使用中文");
|
||||
const en = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, lang: "en" });
|
||||
expect(en).toStartWith("OUTPUT LANGUAGE: ENGLISH");
|
||||
// The directive has to survive a whole Chinese template below it.
|
||||
expect(en).toContain("do not mirror their language");
|
||||
// Structural labels follow the language too — they used to be frozen in
|
||||
// Chinese so downstream parsers could find them, which is what produced
|
||||
// English prose on a Chinese skeleton. The directive now names the English
|
||||
// spelling of every label instead, and the rendered rules use it.
|
||||
expect(en).toContain("复现条件 → Reproduction conditions");
|
||||
expect(en).toContain("期望与实际 → Expected vs actual");
|
||||
expect(en).toContain("## Expected vs actual");
|
||||
expect(en).not.toContain("## 期望与实际 //");
|
||||
// The directive comes FIRST, before the template's own Chinese rules.
|
||||
expect(en.indexOf("OUTPUT LANGUAGE")).toBeLessThan(en.indexOf("铁律"));
|
||||
});
|
||||
|
||||
test("a template that dropped {{langDirective}} renders no directive at all", () => {
|
||||
// The guard rail for the prompt library: the directive is rendered by CODE,
|
||||
// so it governs whichever version agent_prompts has active — but only while
|
||||
// that version keeps the placeholder. A candidate template that drops it
|
||||
// silently sends every en-selected spec back to Chinese.
|
||||
const stripped = DEFAULT_EVAL_PROMPT_TEMPLATE.replace("{{langDirective}}", "");
|
||||
expect(renderEvalPrompt(stripped, { ...vars, lang: "en" })).not.toContain("OUTPUT LANGUAGE");
|
||||
});
|
||||
|
||||
test("an absent remark leaves no empty scaffold behind", () => {
|
||||
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
|
||||
expect(out).not.toContain("Reporter remark");
|
||||
expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, remark: "需要 abid 7712334" }))
|
||||
.toContain("Reporter remark / supplemental info:\n需要 abid 7712334");
|
||||
});
|
||||
|
||||
test("empty fields degrade to (none), never to a blank line", () => {
|
||||
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, description: "", reproSteps: "" });
|
||||
expect(out).toContain("Description (OriginalIntro):\n(none)");
|
||||
expect(out).toContain("Reproduction steps:\n(none)");
|
||||
});
|
||||
|
||||
test("a `$&` in the ticket cannot corrupt the prompt", () => {
|
||||
// Replacement-pattern syntax in user text is the classic silent corruption:
|
||||
// string.replace would expand `$&` into the matched placeholder.
|
||||
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, description: "价格显示 $& 和 $1" });
|
||||
expect(out).toContain("价格显示 $& 和 $1");
|
||||
});
|
||||
|
||||
test("a hand-edited template with a typo'd placeholder shows the typo", () => {
|
||||
// Blanking unknown placeholders would hide the mistake inside a prompt that
|
||||
// still looks plausible. Not `toBe` any more: a platform-specific device
|
||||
// block is appended after the template (see the next test), so the contract
|
||||
// is "the template renders unchanged at the head", not "nothing follows".
|
||||
const out = renderEvalPrompt("头部 {{titel}} / {{title}}", vars);
|
||||
expect(out.startsWith(`头部 {{titel}} / ${vars.title}`)).toBe(true);
|
||||
expect(out).toContain("{{titel}}");
|
||||
});
|
||||
|
||||
/**
|
||||
* 评估期也要拿到设备块。
|
||||
*
|
||||
* 生产的 `*.eval` 模板存在 `agent_prompts` 里,没有一个版本带设备占位符 —— 只改内置
|
||||
* 默认模板对生产是零效果。所以块是**渲染之后追加**的(shared/src/device_prompt.ts 的
|
||||
* ensureDeviceBlockInPrompt),这几条钉的就是这条兜底路径:平台对得上才给、给了不重复、
|
||||
* 别的平台一个字都不多。
|
||||
*/
|
||||
test("设备块按平台追加,且不重复", () => {
|
||||
const android = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
|
||||
expect(android).toContain("rax ttmock boe --ppe");
|
||||
expect(android).toContain("enablePPE");
|
||||
// 追加只发生一次,即便模板自己已经带了占位符。
|
||||
const withPlaceholder = renderEvalPrompt("{{deviceBlock}}\n尾部", vars);
|
||||
expect(withPlaceholder.split("rax ttmock boe --ppe").length - 1).toBe(1);
|
||||
expect(withPlaceholder).toContain("尾部");
|
||||
});
|
||||
|
||||
test("iOS 拿 catuse 那块,web 之类的平台什么都不拿", () => {
|
||||
const ios = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, platform: "ios" });
|
||||
expect(ios).toContain("Skill(catuse-cli)");
|
||||
expect(ios).not.toContain("rax ttmock");
|
||||
const other = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, platform: "web" });
|
||||
expect(other).not.toContain("rax ttmock");
|
||||
expect(other).not.toContain("catuse");
|
||||
});
|
||||
});
|
||||
|
||||
describe("regen prompt rendering", () => {
|
||||
const ctx = { prevPlan: "上一版内容", comments: [{ quote: "复现步骤", comment: "写细一点" }] };
|
||||
|
||||
test("inline comments are numbered and quoted back", () => {
|
||||
const out = renderRegenPrompt(DEFAULT_REGEN_PROMPT_TEMPLATE, { ...vars, ...ctx });
|
||||
expect(out).toContain("1. 【原文片段】「复现步骤」");
|
||||
expect(out).toContain("【人工要求】写细一点");
|
||||
expect(out).toContain("上一版内容");
|
||||
});
|
||||
|
||||
test("no comments still asks for a revision rather than a redraft", () => {
|
||||
const out = renderRegenPrompt(DEFAULT_REGEN_PROMPT_TEMPLATE, { ...vars, prevPlan: "", comments: [] });
|
||||
expect(out).toContain("本次没有逐条批注");
|
||||
expect(out).toContain("(empty)");
|
||||
});
|
||||
|
||||
test("defaultTemplateFor picks the right baseline per stage", () => {
|
||||
expect(defaultTemplateFor("eval")).toBe(DEFAULT_EVAL_PROMPT_TEMPLATE);
|
||||
expect(defaultTemplateFor("regen")).toBe(DEFAULT_REGEN_PROMPT_TEMPLATE);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user