Files
streamline/satellite/test/prompt_fixture_cases.ts
2026-11-16 17:26:44 -08:00

307 lines
13 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* 执行期提示词的夹具输入矩阵 —— 抓取脚本与一致性测试共用同一份定义。
*
* 共用是刻意的。如果抓取用一组输入、测试用另一组,那测试断言的就不是「重构没改措辞」,
* 而是「两组输入碰巧渲染成同一个东西」。这个文件就是那份契约。
*
* 输入矩阵按**分支**设计,不是按数量:每个构造函数里的每个 `? :` 都至少要有一个用例
* 走到两边 —— 首次 / 再次执行、有没有执行期修正、中文 / 英文、有没有外链、有没有复核
* 历史、有没有连着设备。分支没被覆盖到的模板改错了,夹具照样全绿。
*/
import path from "node:path";
import type { Task, Feature, Security, ReviewNote } from "@bbbug/shared/types";
export const FIXTURE_DIR = "satellite/test/fixtures/prompts";
/**
* Environment the builders read. Pinned so a fixture captured on one machine
* matches the one rendered on another — `AI_WORKSPACE` alone defaults to
* `$HOME/ai_workspace`, which is different for every developer.
*/
export const PROMPT_FIXTURE_ENV: Record<string, string> = {
AI_WORKSPACE: "/fixture/ai_workspace",
BASE_BRANCH: "develop",
MAX_ROUNDS: "5",
CLAUDE_BIN: "claude",
SATELLITE_ENTRY_PATH: "/fixture/satellite.js",
STREAMLINE_AGENT_MODE: "claude",
STREAMLINE_DRY_RUN: "1",
};
/** `process.execPath` also appears in generated helper commands, so pin it too. */
export const PROMPT_FIXTURE_EXEC_PATH = "/opt/homebrew/lib/node_modules/bun/bin/bun.exe";
const WS = "/fixture/workspace/task-1";
const REVIEW: ReviewNote[] = [
{ at: 1_760_000_000_000, source: "human", round: 1, text: "红点还在,冷启动后复现" },
{ at: 1_760_000_100_000, source: "critic", round: 1, text: "SCENARIO_BANDAID:直接在该场景 return 了" },
];
const baseTask = {
id: "task-1",
meegoUrl: "https://meego.example/7328764565",
title: "Story 相机特效面板打不开",
platform: "android",
bugType: "UI",
difficulty: "medium",
attempts: 1,
description: "点击特效按钮无反应",
reproSteps: "1. 打开 Story\n2. 点击特效",
lang: "zh",
reviewHistory: [],
reviewFeedback: null,
} as unknown as Task;
const baseFeature = {
id: "feat-1",
title: "消息盒子支持置顶",
platform: "android",
attempts: 1,
description: "希望能把会话置顶",
remark: null,
lang: "zh",
reviewHistory: [],
reviewFeedback: null,
} as unknown as Feature;
const baseSecurity = {
id: "sec-1",
title: "剪贴板越权读取",
platform: "android",
attempts: 1,
reviewSpec: "检查剪贴板读取是否有前台校验",
reproSpec: "冷启动后读取剪贴板",
targetRef: null,
lang: "zh",
reviewHistory: [],
reviewFeedback: null,
} as unknown as Security;
/** A ticket body carrying both link kinds. Figma file keys must be ≥10 chars or the detector ignores them. */
const LINKED = "见 https://bytedance.larkoffice.com/docx/AbCdEf123456 与 https://www.figma.com/design/Q2t0GgS4vT5SKEQ/spec";
export interface FixtureCase { name: string; render: () => string }
/**
* Built lazily and via dynamic import so the env above is already applied when
* the satellite modules read it at module scope (`const BASE_BRANCH = ...`).
* A static import would evaluate before the caller could set anything.
*/
export async function promptFixtureCases(): Promise<FixtureCase[]> {
const runner = await import("../src/runner");
const featureEvaluator = await import("../src/feature_evaluator");
const securityEvaluator = await import("../src/security_evaluator");
const featureRunner = await import("../src/feature_runner");
const securityRunner = await import("../src/security_runner");
const asking = await import("../src/asking");
const cases: FixtureCase[] = [];
// ── bug.run — the longest prompt on the platform, and the most branchy ────
for (const hasReport of [false, true]) {
for (const corrections of [0, 3]) {
for (const lang of ["zh", "en"] as const) {
const t = { ...baseTask, lang } as Task;
cases.push({
name: `bug.run__report-${hasReport ? "y" : "n"}__corr-${corrections}__${lang}`,
render: () => runner.buildSinglePrompt(t, WS, hasReport, corrections),
});
}
}
}
cases.push({
name: "bug.run__with-links",
render: () => runner.buildSinglePrompt(
{ ...baseTask, description: LINKED } as Task, WS, false, 0),
});
cases.push({
name: "bug.run__with-review-history",
render: () => runner.buildSinglePrompt(
{ ...baseTask, reviewHistory: REVIEW } as Task, WS, true, 1),
});
cases.push({
name: "bug.run__legacy-review-feedback",
render: () => runner.buildSinglePrompt(
{ ...baseTask, reviewHistory: [], reviewFeedback: "旧字段:仍然复现" } as Task, WS, true, 0),
});
// 特性验收 bug:基线是 feature 分支而不是主干。走的是 baseBranchFor / baseRefNote
// 两个分支的另一边 —— 首修的检出目标、基线说明段、以及 {{baseBranch}} 本身。
cases.push({
name: "bug.run__base-ref",
render: () => runner.buildSinglePrompt(
{ ...baseTask, baseRef: "feature/im/pin-chat" } as Task, WS, false, 0),
});
// 再修时不走首修的检出段(接着原分支叠加),但 {{baseBranch}} 仍然要跟随基线。
cases.push({
name: "bug.run__base-ref__refix",
render: () => runner.buildSinglePrompt(
{ ...baseTask, baseRef: "feature/im/pin-chat" } as Task, WS, true, 0),
});
// 无工单的 bug:{{meegoUrl}} 落到 meegoUrlForPrompt 的兜底措辞上,而不是空串或
// "null" —— 那一行读起来会像"链接没取到",而不是"本来就没有单"。
cases.push({
name: "bug.run__no-ticket",
render: () => runner.buildSinglePrompt(
{ ...baseTask, meegoId: null, meegoUrl: null } as Task, WS, false, 0),
});
// 派发时带上了注入知识:多两处分支 —— 顶部的知识块与 KB_USED 契约,以及执行条件
// 记录里的 `--kb` 一条。后者只在"这一轮真的有 K 编号可指"时出现;一个被要求标注
// 它从没见过的编号的 agent 会把编号编出来,而编造的关联会喂给归因回环一个假判定。
cases.push({
name: "bug.run__with-kb",
render: () => runner.buildSinglePrompt(
baseTask as Task, WS, false, 0, undefined,
"## 历史真机确证的执行环境知识(参考资料,非权威)\n\n- `K1` [置信 0.90 · 最近验证 2026-08-01] **android:构建 › 远程构建产物**:必须用 ./start.sh 拉包\n"),
});
// ── bug.critic ────────────────────────────────────────────────────────────
// 基线非空时 critic 的 diff 基准要跟着变 —— 基准错了,它会把特性分支上别人的
// 改动一并算进这次修复。
cases.push({
name: "bug.critic__base-ref",
render: () => runner.buildCriticPrompt({ ...baseTask, baseRef: "feature/im/pin-chat" } as Task, WS),
});
for (const lang of ["zh", "en"] as const) {
cases.push({
name: `bug.critic__${lang}`,
render: () => runner.buildCriticPrompt({ ...baseTask, lang } as Task, WS),
});
}
// ── feature.run.first / second ────────────────────────────────────────────
for (const corrections of [0, 2]) {
for (const lang of ["zh", "en"] as const) {
const f = { ...baseFeature, lang } as Feature;
cases.push({
name: `feature.run.first__corr-${corrections}__${lang}`,
render: () => featureRunner.buildFeatureFirstPrompt(f, WS, corrections),
});
cases.push({
name: `feature.run.second__corr-${corrections}__${lang}`,
render: () => featureRunner.buildFeatureSecondPrompt(f, WS, corrections),
});
}
}
cases.push({
name: "feature.run.first__with-links-and-review",
render: () => featureRunner.buildFeatureFirstPrompt(
{ ...baseFeature, description: LINKED, reviewHistory: REVIEW } as Feature, WS, 0),
});
// ── security.run.first / second ───────────────────────────────────────────
for (const corrections of [0, 2]) {
cases.push({
name: `security.run.first__corr-${corrections}`,
render: () => securityRunner.buildSecurityFirstPrompt(baseSecurity, WS, corrections),
});
cases.push({
name: `security.run.second__corr-${corrections}`,
render: () => securityRunner.buildSecuritySecondPrompt(baseSecurity, WS, corrections),
});
}
cases.push({
name: "security.run.first__target-ref",
render: () => securityRunner.buildSecurityFirstPrompt(
{ ...baseSecurity, targetRef: "release/8.1", reviewHistory: REVIEW } as Security, WS, 0),
});
cases.push({
name: "security.run.first__en",
render: () => securityRunner.buildSecurityFirstPrompt(
{ ...baseSecurity, lang: "en" } as Security, WS, 0),
});
// ── review.asking — one per kind; the report file is absent on purpose so the
// "(报告文件尚不存在)" branch is the one pinned (reading a real file would
// make the fixture depend on a directory that is not in the repo).
for (const kind of ["bug", "feature", "security"] as const) {
cases.push({
name: `review.asking__${kind}`,
render: () => asking.buildPreamble({
key: `${kind}:1`,
kind,
dir: path.join("/fixture/taskLog", `${kind}-1`),
branch: "bugfix/story-camera-effect",
context: {
title: "Story 相机特效面板打不开",
meegoUrl: "https://meego.example/7328764565",
platform: "android",
},
} as Parameters<typeof asking.buildPreamble>[0]),
});
}
// ── feature.eval / feature.regen ──────────────────────────────────────────
for (const lang of ["zh", "en"] as const) {
cases.push({
name: `feature.eval__${lang}`,
render: () => featureEvaluator.buildFeatureEvalPrompt({ ...baseFeature, lang } as Feature),
});
}
cases.push({
name: "feature.eval__remark",
render: () => featureEvaluator.buildFeatureEvalPrompt(
{ ...baseFeature, remark: "补充:只在国内版生效" } as Feature),
});
cases.push({
// The cross-platform port brief takes a different remark branch entirely.
name: "feature.eval__port-brief",
render: () => featureEvaluator.buildFeatureEvalPrompt(
{ ...baseFeature, remark: "对端已验收的实现要点…", portSourceId: "feat-src-1" } as Feature),
});
cases.push({
name: "feature.eval__with-links",
render: () => featureEvaluator.buildFeatureEvalPrompt(
{ ...baseFeature, description: LINKED } as Feature),
});
const regenCtx = {
prevPlan: "## 目标\n- 支持置顶\n\n## 验收标准\n- 能置顶",
comments: [
{ quote: "支持置顶", comment: "要写清最多能置顶几个" },
{ quote: "能置顶", comment: "补一条取消置顶的验收" },
],
};
for (const lang of ["zh", "en"] as const) {
cases.push({
name: `feature.regen__${lang}`,
render: () => featureEvaluator.buildFeatureRegenPrompt(
{ ...baseFeature, lang } as Feature,
regenCtx as Parameters<typeof featureEvaluator.buildFeatureRegenPrompt>[1]),
});
}
cases.push({
// No inline comments → the "整体复核" fallback line, a distinct branch.
name: "feature.regen__no-comments",
render: () => featureEvaluator.buildFeatureRegenPrompt(
{ ...baseFeature, remark: "补充说明" } as Feature,
{ prevPlan: "## 目标\n- 旧稿", comments: [] } as Parameters<typeof featureEvaluator.buildFeatureRegenPrompt>[1]),
});
cases.push({
name: "feature.regen__empty-prev-plan",
render: () => featureEvaluator.buildFeatureRegenPrompt(
baseFeature,
{ prevPlan: "", comments: regenCtx.comments } as Parameters<typeof featureEvaluator.buildFeatureRegenPrompt>[1]),
});
// ── security.eval ─────────────────────────────────────────────────────────
for (const lang of ["zh", "en"] as const) {
cases.push({
name: `security.eval__${lang}`,
render: () => securityEvaluator.buildSecurityEvalPrompt({ ...baseSecurity, lang } as Security, WS),
});
}
cases.push({
name: "security.eval__target-ref",
render: () => securityEvaluator.buildSecurityEvalPrompt(
{ ...baseSecurity, targetRef: "release/8.1" } as Security, WS),
});
cases.push({
name: "security.eval__remark",
render: () => securityEvaluator.buildSecurityEvalPrompt(
{ ...baseSecurity, remark: "只在越南地区复现" } as Security, WS),
});
return cases;
}