Initial import of streamline

This commit is contained in:
2026-11-16 17:26:44 -08:00
commit 8e4aec1855
1221 changed files with 274256 additions and 0 deletions
+166
View File
@@ -0,0 +1,166 @@
import { test, expect, describe } from "bun:test";
import {
DEFAULT_EVAL_PROMPT_TEMPLATE, DEFAULT_REGEN_PROMPT_TEMPLATE,
renderEvalPrompt, renderRegenPrompt, defaultTemplateFor,
} from "@bbbug/shared/eval_prompt";
import { NEEDS_INPUT_OPEN } from "@bbbug/shared/protocol";
/**
* These prompts moved out of satellite/src/evaluator.ts into a shared template so
* they could be versioned, handed down at dispatch, and replayed offline. At
* migration time the rendered output was diffed against the old inline builders
* across 24 combinations (lang × remark × kbContext × regen) and came out
* byte-identical — the English language directive was strengthened deliberately
* afterwards, and is the one intended difference.
*
* What these tests pin is the CONTRACT that made the parity possible, so a later
* edit to a template (by hand, or by an approved prompt strategy) cannot quietly
* break a downstream parser: the ASSESSMENT/<PLAN> sentinels the evaluator reads,
* the conditional KB_USED line, and the structural labels the replay scorer and
* the fix prompt locate sections by.
*/
const vars = {
lang: "zh" as const,
meegoUrl: "https://meego.example/7328764565",
title: "Story 相机特效面板打不开",
platform: "android",
bugType: "UI",
description: "点击特效按钮无反应",
reproSteps: "1. 打开 Story\n2. 点击特效",
remark: null as string | null,
kbContext: null as string | null,
};
const KB = "## 已验证的历史复现知识(参考,非权威)\n- [置信 0.87] Story 相机:入口在右上角";
describe("eval prompt rendering", () => {
test("every placeholder is substituted", () => {
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
expect(out).not.toContain("{{");
expect(out).toContain(vars.meegoUrl);
expect(out).toContain(vars.title);
});
test("the output contract the evaluator parses is present", () => {
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
expect(out).toContain("ASSESSMENT: <REPRODUCIBLE | NEEDS_INFO>");
expect(out).toContain("<PLAN>");
expect(out).toContain("</PLAN>");
expect(out).toContain(NEEDS_INPUT_OPEN);
});
test("KB_USED is asked for ONLY when something was injected", () => {
// Asking for a citation with no reference block invites invented labels, and
// an invented K2 scores a hit against a claim that was never shown.
expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars)).not.toContain("KB_USED");
const withKb = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, kbContext: KB });
expect(withKb).toContain("KB_USED");
expect(withKb).toContain("入口在右上角");
});
test("language directive follows the submitter's choice", () => {
expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars)).toStartWith("请使用中文");
const en = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, lang: "en" });
expect(en).toStartWith("OUTPUT LANGUAGE: ENGLISH");
// The directive has to survive a whole Chinese template below it.
expect(en).toContain("do not mirror their language");
// Structural labels follow the language too — they used to be frozen in
// Chinese so downstream parsers could find them, which is what produced
// English prose on a Chinese skeleton. The directive now names the English
// spelling of every label instead, and the rendered rules use it.
expect(en).toContain("复现条件 → Reproduction conditions");
expect(en).toContain("期望与实际 → Expected vs actual");
expect(en).toContain("## Expected vs actual");
expect(en).not.toContain("## 期望与实际 //");
// The directive comes FIRST, before the template's own Chinese rules.
expect(en.indexOf("OUTPUT LANGUAGE")).toBeLessThan(en.indexOf("铁律"));
});
test("a template that dropped {{langDirective}} renders no directive at all", () => {
// The guard rail for the prompt library: the directive is rendered by CODE,
// so it governs whichever version agent_prompts has active — but only while
// that version keeps the placeholder. A candidate template that drops it
// silently sends every en-selected spec back to Chinese.
const stripped = DEFAULT_EVAL_PROMPT_TEMPLATE.replace("{{langDirective}}", "");
expect(renderEvalPrompt(stripped, { ...vars, lang: "en" })).not.toContain("OUTPUT LANGUAGE");
});
test("an absent remark leaves no empty scaffold behind", () => {
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
expect(out).not.toContain("Reporter remark");
expect(renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, remark: "需要 abid 7712334" }))
.toContain("Reporter remark / supplemental info:\n需要 abid 7712334");
});
test("empty fields degrade to (none), never to a blank line", () => {
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, description: "", reproSteps: "" });
expect(out).toContain("Description (OriginalIntro):\n(none)");
expect(out).toContain("Reproduction steps:\n(none)");
});
test("a `$&` in the ticket cannot corrupt the prompt", () => {
// Replacement-pattern syntax in user text is the classic silent corruption:
// string.replace would expand `$&` into the matched placeholder.
const out = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, description: "价格显示 $& 和 $1" });
expect(out).toContain("价格显示 $& 和 $1");
});
test("a hand-edited template with a typo'd placeholder shows the typo", () => {
// Blanking unknown placeholders would hide the mistake inside a prompt that
// still looks plausible. Not `toBe` any more: a platform-specific device
// block is appended after the template (see the next test), so the contract
// is "the template renders unchanged at the head", not "nothing follows".
const out = renderEvalPrompt("头部 {{titel}} / {{title}}", vars);
expect(out.startsWith(`头部 {{titel}} / ${vars.title}`)).toBe(true);
expect(out).toContain("{{titel}}");
});
/**
* 评估期也要拿到设备块。
*
* 生产的 `*.eval` 模板存在 `agent_prompts` 里,没有一个版本带设备占位符 —— 只改内置
* 默认模板对生产是零效果。所以块是**渲染之后追加**的(shared/src/device_prompt.ts 的
* ensureDeviceBlockInPrompt),这几条钉的就是这条兜底路径:平台对得上才给、给了不重复、
* 别的平台一个字都不多。
*/
test("设备块按平台追加,且不重复", () => {
const android = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, vars);
expect(android).toContain("rax ttmock boe --ppe");
expect(android).toContain("enablePPE");
// 追加只发生一次,即便模板自己已经带了占位符。
const withPlaceholder = renderEvalPrompt("{{deviceBlock}}\n尾部", vars);
expect(withPlaceholder.split("rax ttmock boe --ppe").length - 1).toBe(1);
expect(withPlaceholder).toContain("尾部");
});
test("iOS 拿 catuse 那块,web 之类的平台什么都不拿", () => {
const ios = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, platform: "ios" });
expect(ios).toContain("Skill(catuse-cli)");
expect(ios).not.toContain("rax ttmock");
const other = renderEvalPrompt(DEFAULT_EVAL_PROMPT_TEMPLATE, { ...vars, platform: "web" });
expect(other).not.toContain("rax ttmock");
expect(other).not.toContain("catuse");
});
});
describe("regen prompt rendering", () => {
const ctx = { prevPlan: "上一版内容", comments: [{ quote: "复现步骤", comment: "写细一点" }] };
test("inline comments are numbered and quoted back", () => {
const out = renderRegenPrompt(DEFAULT_REGEN_PROMPT_TEMPLATE, { ...vars, ...ctx });
expect(out).toContain("1. 【原文片段】「复现步骤」");
expect(out).toContain("【人工要求】写细一点");
expect(out).toContain("上一版内容");
});
test("no comments still asks for a revision rather than a redraft", () => {
const out = renderRegenPrompt(DEFAULT_REGEN_PROMPT_TEMPLATE, { ...vars, prevPlan: "", comments: [] });
expect(out).toContain("本次没有逐条批注");
expect(out).toContain("(empty)");
});
test("defaultTemplateFor picks the right baseline per stage", () => {
expect(defaultTemplateFor("eval")).toBe(DEFAULT_EVAL_PROMPT_TEMPLATE);
expect(defaultTemplateFor("regen")).toBe(DEFAULT_REGEN_PROMPT_TEMPLATE);
});
});