Initial import of streamline
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Unit tests for the standalone RSI loop."""
|
||||
@@ -0,0 +1,481 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from unittest import mock
|
||||
|
||||
from rsi_loop import config, episodes, llm, mine, proposal, replay, runner, runtime, scoring, transcript
|
||||
from rsi_loop.runner import _eligible_cluster
|
||||
|
||||
LAB_NODES = {"android": "evaluator-android-01.shiro0.icu", "ios": "temp-ios-01"}
|
||||
|
||||
|
||||
def assistant(tool_id, name, tool_input, tokens=1000):
|
||||
return {
|
||||
"type": "assistant",
|
||||
"message": {
|
||||
"usage": {"input_tokens": tokens},
|
||||
"content": [{"type": "tool_use", "id": tool_id, "name": name, "input": tool_input}],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def result(tool_id, text, is_error=False, timestamp="2026-09-01T00:00:00Z"):
|
||||
return {
|
||||
"type": "user", "timestamp": timestamp,
|
||||
"message": {"content": [{
|
||||
"type": "tool_result", "tool_use_id": tool_id,
|
||||
"content": text, "is_error": is_error,
|
||||
}]},
|
||||
}
|
||||
|
||||
|
||||
class TranscriptTests(unittest.TestCase):
|
||||
def test_read_only_output_does_not_become_failure(self):
|
||||
rows = [
|
||||
assistant("a", "Bash", {"command": "grep -R error src"}),
|
||||
result("a", "error: source fixture\nExit code 1"),
|
||||
]
|
||||
call = transcript.calls(rows)[0]
|
||||
self.assertTrue(call["read_only"])
|
||||
self.assertTrue(call["ok"])
|
||||
self.assertIsNone(call["wall"])
|
||||
|
||||
def test_repeated_failure_point_is_after_first_failure(self):
|
||||
rows = [
|
||||
assistant("a", "Bash", {"command": "adb install app.apk"}),
|
||||
result("a", "INSTALL_FAILED_VERSION_DOWNGRADE", True),
|
||||
assistant("b", "Bash", {"command": "adb install app.apk"}),
|
||||
result("b", "INSTALL_FAILED_VERSION_DOWNGRADE", True),
|
||||
]
|
||||
found = episodes.find_episodes(rows)
|
||||
self.assertEqual(len(found), 1)
|
||||
episode = found[0]
|
||||
self.assertEqual(episode["n_fail"], 2)
|
||||
self.assertEqual(episode["point_call"], 0)
|
||||
self.assertEqual(episode["point_row"], 1)
|
||||
|
||||
def test_wait_point_is_before_wait_command(self):
|
||||
rows = [
|
||||
assistant("a", "Bash", {"command": "adb shell input tap 1 2"}),
|
||||
result("a", "ok"),
|
||||
assistant("b", "Bash", {"command": "until adb shell dumpsys window; do sleep 5; done"}),
|
||||
result("b", "done"),
|
||||
]
|
||||
point = episodes.harvest_wait_points(rows)[0]
|
||||
self.assertEqual(point["point_call"], 0)
|
||||
self.assertEqual(point["point_row"], 1)
|
||||
|
||||
|
||||
class ProposalTests(unittest.TestCase):
|
||||
def test_only_inserts_and_preserves_placeholders(self):
|
||||
base = "A {{task}}\nB\nC {{round}}"
|
||||
candidate = proposal.apply_insertion(base, "新增一\n新增二", "B")
|
||||
self.assertEqual(candidate, "A {{task}}\nB\n新增一\n新增二\nC {{round}}")
|
||||
self.assertIsNone(proposal.candidate_error(base, candidate))
|
||||
self.assertEqual(proposal.inserted_text(base, candidate), "新增一\n新增二")
|
||||
|
||||
def test_rejects_changed_line_and_ambiguous_anchor(self):
|
||||
base = "A\nB\nB"
|
||||
with self.assertRaises(ValueError):
|
||||
proposal.apply_insertion(base, "X", "B")
|
||||
self.assertIsNotNone(proposal.candidate_error("A\nB", "A\nB changed"))
|
||||
|
||||
def test_reach_target_cannot_touch_evidence_capture(self):
|
||||
base = "步骤 3\n复现问题。\n步骤 4"
|
||||
shoot = proposal.apply_insertion(base, "- 启动 App 后立刻 record-evidence 拍一张 before-1-x.png", "复现问题。")
|
||||
route = proposal.apply_insertion(base, "- 先按 TestSpec 的入口用 deeplink 直达目标页", "复现问题。")
|
||||
self.assertIsNotNone(proposal.gaming_error("reach_s", base, shoot))
|
||||
self.assertIsNotNone(proposal.gaming_error("reach_miss", base, shoot))
|
||||
self.assertIsNone(proposal.gaming_error("reach_s", base, route))
|
||||
self.assertIsNone(proposal.gaming_error("evidence_fail", base, shoot))
|
||||
|
||||
def test_business_metrics_are_registrable_for_every_replay_type(self):
|
||||
for ptype, allowed in proposal.METRICS_BY_REPLAY.items():
|
||||
self.assertTrue(allowed & proposal.BUSINESS_METRICS, ptype)
|
||||
self.assertTrue(allowed <= proposal.TARGET_METRICS, ptype)
|
||||
|
||||
def test_nested_json_reply_is_parsed(self):
|
||||
with mock.patch.object(llm, "ask", return_value='```json\n{"a":{"b":1}}\n```'):
|
||||
self.assertEqual(llm.ask_json("x"), {"a": {"b": 1}})
|
||||
|
||||
def test_online_metric_must_match_offline_action_shape(self):
|
||||
value = {
|
||||
"title": "x", "summary": "x", "hypothesis": "x", "method": "x",
|
||||
"targetMetric": "tool_calls", "afterLine": None, "content": "新增纪律",
|
||||
}
|
||||
cluster = {
|
||||
"stage": "bug.run", "problemClass": "decision_error",
|
||||
"cluster": "wait", "clusterTitle": "等待", "replayType": "T2_wait",
|
||||
"wasteCalls": 3, "episodes": 1, "transcripts": 1, "samples": [],
|
||||
}
|
||||
with mock.patch.object(llm, "ask_json", return_value=value):
|
||||
with self.assertRaisesRegex(ValueError, "不对应"):
|
||||
proposal.create(cluster, "base")
|
||||
|
||||
|
||||
class ScoringTests(unittest.TestCase):
|
||||
def test_action_shape_rules(self):
|
||||
self.assertEqual(scoring.classify_action(
|
||||
"T2_wait", {}, {"name": "Bash", "input": {"command": "sleep 30"}},
|
||||
), "bad")
|
||||
self.assertEqual(scoring.classify_action(
|
||||
"T2_wait", {}, {"name": "Bash", "input": {"command": "node wait-for.js"}},
|
||||
), "good")
|
||||
self.assertEqual(scoring.classify_action(
|
||||
"T3_frida", {}, {"name": "Bash", "input": {"command": "frida -U -n app -l hook.js"}},
|
||||
), "bad")
|
||||
self.assertEqual(scoring.classify_action(
|
||||
"T3_frida", {}, {"name": "Bash", "input": {"command": "frida --version"}},
|
||||
), "good")
|
||||
point = {"fam": "adb install app.apk"}
|
||||
self.assertEqual(scoring.classify_action(
|
||||
"T1_repeat", point, {"name": "Bash", "input": {"command": "adb install app.apk"}},
|
||||
), "bad")
|
||||
self.assertEqual(scoring.classify_action(
|
||||
"T1_repeat", point, {"name": "Bash", "input": {"command": "adb shell getprop"}},
|
||||
), "good")
|
||||
|
||||
def test_paired_bootstrap_is_deterministic_and_positive(self):
|
||||
records = []
|
||||
for point in range(20):
|
||||
for seed in range(2):
|
||||
records.append({"pointId": str(point), "arm": "base", "seed": seed, "classification": "bad"})
|
||||
records.append({"pointId": str(point), "arm": "candidate", "seed": seed, "classification": "good"})
|
||||
a = scoring.offline_summary(records)
|
||||
b = scoring.offline_summary(records)
|
||||
self.assertEqual(a, b)
|
||||
self.assertEqual(a["points"], 20)
|
||||
self.assertEqual(a["seeds"], 2)
|
||||
self.assertGreater(a["ciLow"], 0)
|
||||
|
||||
|
||||
class AuditAndSelectionTests(unittest.TestCase):
|
||||
def test_audit_requires_verbatim_evidence(self):
|
||||
raw = {
|
||||
"classification": "decision_error", "cluster": "repeat_same_command",
|
||||
"clusterTitle": "重复撞墙", "replayType": "T1_repeat",
|
||||
"correctInfoStep": 3, "evidence": ["Exit code 1"], "rationale": "已出现错误",
|
||||
}
|
||||
self.assertIsNotNone(mine._valid_audit(raw, "tool result: Exit code 1"))
|
||||
raw["evidence"] = ["模型编造的引文"]
|
||||
self.assertIsNone(mine._valid_audit(raw, "tool result: Exit code 1"))
|
||||
|
||||
def test_live_stage_and_cooldown_are_skipped(self):
|
||||
now = 2_000_000_000_000
|
||||
clusters = [
|
||||
{"stage": "bug.run", "platform": "android", "cluster": "a", "replayType": "T1_repeat"},
|
||||
{"stage": "bug.run", "platform": "ios", "cluster": "b", "replayType": "T2_wait"},
|
||||
{"stage": "bug.run", "platform": "ios", "cluster": "c", "replayType": "T3_frida"},
|
||||
]
|
||||
state = {
|
||||
"labNodes": LAB_NODES,
|
||||
"live": [{"stage": "bug.run", "platform": "android"}],
|
||||
"recent": [{"stage": "bug.run", "platform": "ios", "cluster": "b", "createdAt": now}],
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(config, "DATA", Path(d)):
|
||||
self.assertEqual(_eligible_cluster(clusters, state, now, point_count=lambda c: 30)["cluster"], "c")
|
||||
|
||||
def test_cluster_without_enough_points_is_skipped(self):
|
||||
now = 2_000_000_000_000
|
||||
clusters = [
|
||||
{"stage": "bug.run", "platform": "android", "cluster": "few", "replayType": "T1_repeat"},
|
||||
{"stage": "bug.run", "platform": "android", "cluster": "many", "replayType": "T1_repeat"},
|
||||
]
|
||||
counts = {"few": 5, "many": 40}
|
||||
state = {"labNodes": LAB_NODES}
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(config, "DATA", Path(d)):
|
||||
got = _eligible_cluster(clusters, state, now, point_count=lambda c: counts[c["cluster"]])
|
||||
self.assertEqual(got["cluster"], "many")
|
||||
runner._record_skip(clusters[1], "anchor missing", now)
|
||||
self.assertIsNone(_eligible_cluster(clusters, state, now, point_count=lambda c: counts[c["cluster"]]))
|
||||
|
||||
|
||||
class ReplayFidelityTests(unittest.TestCase):
|
||||
PROMPT = "你是自主 Bug 修复 Agent。\n工作区在 /Users/fei/bug_workspace/123。\n## 等待\n用 wait-for。\n## 结尾"
|
||||
|
||||
def test_insert_at_anchor_with_placeholder_and_no_heading(self):
|
||||
ins = {"afterLine": "工作区在 {{workspace}}。", "content": "新纪律一\n新纪律二"}
|
||||
out = replay.insert_into_text(self.PROMPT, ins)
|
||||
lines = out.split("\n")
|
||||
self.assertEqual(lines[1], "工作区在 /Users/fei/bug_workspace/123。")
|
||||
self.assertEqual(lines[2:4], ["新纪律一", "新纪律二"])
|
||||
self.assertNotIn("候选", out)
|
||||
|
||||
def test_null_anchor_appends_at_end_and_missing_anchor_raises(self):
|
||||
out = replay.insert_into_text(self.PROMPT, {"afterLine": None, "content": "末尾纪律"})
|
||||
self.assertTrue(out.endswith("## 结尾\n末尾纪律"))
|
||||
with self.assertRaises(replay.AnchorMissing):
|
||||
replay.insert_into_text(self.PROMPT, {"afterLine": "不存在的一行", "content": "x"})
|
||||
|
||||
def test_candidate_applies_checks_first_prompt(self):
|
||||
rows = [{"type": "user", "message": {"role": "user", "content": self.PROMPT}}]
|
||||
self.assertTrue(replay.candidate_applies(rows, {"afterLine": "## 等待", "content": "x"}))
|
||||
self.assertFalse(replay.candidate_applies(rows, {"afterLine": "## 不存在", "content": "x"}))
|
||||
self.assertTrue(replay.candidate_applies(rows, None))
|
||||
|
||||
def test_environment_snapshot_uses_this_runs_cwd(self):
|
||||
rows = [
|
||||
{"type": "user", "message": {"role": "user", "content": self.PROMPT}},
|
||||
{"type": "attachment", "attachment": {"type": "environment",
|
||||
"snapshot": {"workingDirectory": "/Users/fei/ai_workspace"},
|
||||
"changes": [{"field": "workingDirectory"}]}},
|
||||
]
|
||||
loc = {"environment": {"type": "environment", "snapshot": {"workingDirectory": "/probe/dir", "platform": "darwin"}}}
|
||||
out, _ = replay._neutralize(rows, "sid", loc, Path("/runs/abc"), Path("/proj/abc"),
|
||||
{"afterLine": "## 等待", "content": "新纪律"})
|
||||
env = out[1]["attachment"]
|
||||
self.assertEqual(env["snapshot"]["workingDirectory"], "/runs/abc")
|
||||
self.assertNotIn("changes", env)
|
||||
self.assertIn("## 等待\n新纪律\n用 wait-for。", out[0]["message"]["content"])
|
||||
|
||||
|
||||
class ClusterConsolidationTests(unittest.TestCase):
|
||||
def _audits(self, d):
|
||||
rows = [
|
||||
{"episodeId": "e1", "stage": "bug.run", "classification": "decision_error", "cluster": "app_ready_blind_wait", "clusterTitle": "盲等就绪", "replayType": "T2_wait"},
|
||||
{"episodeId": "e2", "stage": "bug.run", "classification": "decision_error", "cluster": "fixed_sleep_before_ready", "clusterTitle": "定长 sleep 等就绪", "replayType": "T2_wait"},
|
||||
{"episodeId": "e3", "stage": "bug.run", "classification": "decision_error", "cluster": "frida_retry", "clusterTitle": "Frida 原样重试", "replayType": "T3_frida"},
|
||||
]
|
||||
(Path(d) / "a.jsonl").write_text("\n".join(json.dumps(r, ensure_ascii=False) for r in rows))
|
||||
|
||||
def test_canon_follows_chain_and_stops_on_cycle(self):
|
||||
self.assertEqual(mine.canon("a", {"a": "b", "b": "c"}), "c")
|
||||
self.assertIn(mine.canon("a", {"a": "b", "b": "a"}), ("a", "b"))
|
||||
self.assertEqual(mine.canon("x", {}), "x")
|
||||
|
||||
def test_consolidation_validates_and_merges(self):
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(config, "DATA", Path(d)), \
|
||||
mock.patch.object(config, "AUDITS", Path(d) / "a.jsonl"):
|
||||
self._audits(d)
|
||||
reply = {"merge": {
|
||||
"fixed_sleep_before_ready": "app_ready_blind_wait", # 同机制、同重放型 → 并
|
||||
"frida_retry": "app_ready_blind_wait", # 重放型不同 → 拒
|
||||
"app_ready_blind_wait": "made_up_name", # 规范名不存在 → 拒
|
||||
}}
|
||||
with mock.patch.object(llm, "ask_json", return_value=reply):
|
||||
r = mine.consolidate_clusters()
|
||||
self.assertEqual(r["merged"], 1)
|
||||
names = {k["cluster"]: k["count"] for k in mine.known_clusters()}
|
||||
self.assertEqual(names, {"app_ready_blind_wait": 2, "frida_retry": 1})
|
||||
|
||||
def test_audit_prompt_lists_known_clusters(self):
|
||||
known = [{"cluster": "app_ready_blind_wait", "clusterTitle": "盲等就绪", "classification": "decision_error", "replayType": "T2_wait", "count": 5}]
|
||||
p = mine._audit_prompt({"stage": "bug.run", "label": "wall:x", "n_fail": 2, "first_call": 1, "last_call": 3}, "时间线", known)
|
||||
self.assertIn("app_ready_blind_wait", p)
|
||||
self.assertIn("必须复用", p)
|
||||
|
||||
|
||||
class LabModeTests(unittest.TestCase):
|
||||
def test_lab_run_sessions_are_not_mined(self):
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(config, "DATA", Path(d)), \
|
||||
mock.patch.object(config, "EPISODES", Path(d) / "e.jsonl"), mock.patch.object(config, "POINTS", Path(d) / "p.jsonl"), \
|
||||
mock.patch.object(mine, "transcript_files", return_value=[(Path(d) / "x.jsonl", {"workspaceKey": "rsi-lab-abc", "node": "n", "file": "x.jsonl"})]), \
|
||||
mock.patch.object(mine, "load_rows", side_effect=AssertionError("复跑会话不该被读")):
|
||||
r = mine.mine_local()
|
||||
self.assertEqual(r["episodes"], 0)
|
||||
|
||||
def test_only_submits_bug_run_with_lab_node(self):
|
||||
now = 2_000_000_000_000
|
||||
clusters = [
|
||||
{"stage": "feature.run.first", "platform": "android", "cluster": "f", "replayType": "T2_wait"},
|
||||
{"stage": "bug.run", "platform": "ios", "cluster": "no_lab_node", "replayType": "T1_repeat"},
|
||||
{"stage": "bug.run", "platform": "android", "cluster": "ok", "replayType": "T2_wait"},
|
||||
]
|
||||
state = {"labNodes": {"android": "evaluator-android-01.shiro0.icu"}}
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(config, "DATA", Path(d)):
|
||||
self.assertEqual(_eligible_cluster(clusters, state, now, point_count=lambda c: 30)["cluster"], "ok")
|
||||
# 没配实验机就一个都不提
|
||||
self.assertIsNone(_eligible_cluster(clusters, {}, now, point_count=lambda c: 30))
|
||||
|
||||
def test_lab_verdicts_get_conclusions(self):
|
||||
self.assertIn("lab_passed", runner.TERMINAL)
|
||||
self.assertIn("lab_failed", runner.TERMINAL)
|
||||
self.assertIn("declined", runner.TERMINAL)
|
||||
|
||||
|
||||
class PlatformTests(unittest.TestCase):
|
||||
def test_live_experiment_only_blocks_its_own_platform(self):
|
||||
now = 2_000_000_000_000
|
||||
clusters = [
|
||||
{"stage": "bug.run", "platform": "android", "cluster": "a", "replayType": "T2_wait"},
|
||||
{"stage": "bug.run", "platform": "ios", "cluster": "b", "replayType": "T2_wait"},
|
||||
]
|
||||
state = {"labNodes": LAB_NODES, "live": [{"stage": "bug.run", "platform": "android"}]}
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(config, "DATA", Path(d)):
|
||||
self.assertEqual(_eligible_cluster(clusters, state, now, point_count=lambda c: 30)["cluster"], "b")
|
||||
# 不带平台的旧实验对所有平台生效
|
||||
legacy = {"labNodes": LAB_NODES, "live": [{"stage": "bug.run", "platform": None}]}
|
||||
self.assertIsNone(_eligible_cluster(clusters, legacy, now, point_count=lambda c: 30))
|
||||
|
||||
def test_replay_points_stay_on_cluster_platform(self):
|
||||
pts = [
|
||||
{"id": "a", "stage": "bug.run", "platform": "android", "ptype": "T2_wait"},
|
||||
{"id": "i", "stage": "bug.run", "platform": "ios", "ptype": "T2_wait"},
|
||||
]
|
||||
with mock.patch.object(mine, "_read_jsonl", return_value=pts):
|
||||
got = mine.replay_points({"stage": "bug.run", "platform": "ios", "replayType": "T2_wait"})
|
||||
self.assertEqual([p["id"] for p in got], ["i"])
|
||||
|
||||
def test_node_without_transcripts_is_not_an_error(self):
|
||||
import urllib.error
|
||||
|
||||
def fake_get(path, timeout=60):
|
||||
if path == "/nodes":
|
||||
return {"nodes": [{"id": "on", "platform": "android"}, {"id": "off", "platform": "ios"}]}
|
||||
if path.startswith("/nodes/off/"):
|
||||
raise urllib.error.HTTPError(path, 404, "not found", {}, None)
|
||||
return {"files": []}
|
||||
|
||||
with tempfile.TemporaryDirectory() as d, \
|
||||
mock.patch.object(config, "DATA", Path(d)), mock.patch.object(config, "TX_DIR", Path(d) / "tx"), \
|
||||
mock.patch.object(config, "RUNS", Path(d) / "runs"), mock.patch.object(config, "STATE", Path(d) / "state.json"), \
|
||||
mock.patch.object(config, "NODES", []), mock.patch("rsi_loop.api.get", side_effect=fake_get):
|
||||
stats = mine.sync_transcripts(2_000_000_000_000)
|
||||
self.assertEqual(stats["nodes"], 1)
|
||||
self.assertEqual(stats["notEnabled"], ["off"])
|
||||
self.assertEqual(stats["errors"], [])
|
||||
|
||||
|
||||
class SyncTests(unittest.TestCase):
|
||||
def test_one_broken_download_does_not_stop_the_rest(self):
|
||||
files = [{"file": f"{i:08d}-0000-0000-0000-000000000000.jsonl", "size": 10, "mtime": 2_000_000_000_000} for i in range(3)]
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_download(path, dest):
|
||||
calls["n"] += 1
|
||||
if "00000001-" in path:
|
||||
raise IOError("IncompleteRead(542248 bytes read)")
|
||||
dest.write_text("{}")
|
||||
return 10
|
||||
|
||||
with tempfile.TemporaryDirectory() as d, \
|
||||
mock.patch.object(config, "DATA", Path(d)), mock.patch.object(config, "TX_DIR", Path(d) / "tx"), \
|
||||
mock.patch.object(config, "RUNS", Path(d) / "runs"), mock.patch.object(config, "STATE", Path(d) / "state.json"), \
|
||||
mock.patch.object(config, "NODES", ["n1"]), \
|
||||
mock.patch("rsi_loop.api.get", return_value={"files": files}), \
|
||||
mock.patch("rsi_loop.api.download", side_effect=fake_download), \
|
||||
mock.patch("rsi_loop.mine.time.sleep"):
|
||||
stats = mine.sync_transcripts(2_000_000_000_000)
|
||||
self.assertEqual(stats["downloaded"], 2)
|
||||
self.assertEqual(len(stats["errors"]), 1)
|
||||
self.assertIn("IncompleteRead", stats["errors"][0])
|
||||
|
||||
|
||||
class DryRunTests(unittest.TestCase):
|
||||
def test_dry_run_post_never_touches_network(self):
|
||||
from rsi_loop import api
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(config, "DATA", Path(d)), \
|
||||
mock.patch.object(config, "DRY_RUN", True), \
|
||||
mock.patch("urllib.request.urlopen", side_effect=AssertionError("dry-run 不该联网")):
|
||||
got = api.post("/experiments", {"title": "t", "stage": "bug.run"})
|
||||
self.assertEqual(got["experiment"]["status"], "dry_run")
|
||||
self.assertTrue(api.event("运行摘要"))
|
||||
lines = (Path(d) / "dry_posts.jsonl").read_text().splitlines()
|
||||
self.assertEqual([json.loads(l)["path"] for l in lines], ["/experiments", "/events"])
|
||||
runner._record_skip({"stage": "bug.run", "cluster": "x"}, "anchor", 1)
|
||||
self.assertFalse((Path(d) / "skips.json").exists())
|
||||
|
||||
|
||||
class FirstTurnTests(unittest.TestCase):
|
||||
def test_text_then_tool_use_is_one_turn(self):
|
||||
lines = [
|
||||
json.dumps({"type": "system", "subtype": "init"}),
|
||||
json.dumps({"type": "assistant", "message": {"id": "m1", "content": [{"type": "text", "text": "先冷启动看登录态。"}]}}),
|
||||
json.dumps({"type": "assistant", "message": {"id": "m1", "content": [
|
||||
{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"command": "adb shell monkey -p x 1"}}]}}),
|
||||
json.dumps({"type": "result", "is_error": False}),
|
||||
]
|
||||
first, err = replay.first_turn(lines)
|
||||
self.assertIsNone(err)
|
||||
self.assertEqual(scoring.action_of({"first": first})["input"]["command"], "adb shell monkey -p x 1")
|
||||
|
||||
def test_error_result_is_reported(self):
|
||||
first, err = replay.first_turn([json.dumps({"type": "result", "is_error": True, "result": "Failed to authenticate"})])
|
||||
self.assertIsNone(first)
|
||||
self.assertIn("authenticate", err)
|
||||
|
||||
|
||||
class SamplingTests(unittest.TestCase):
|
||||
def _points(self):
|
||||
mk = lambda i, stage, ptype, ep: {"id": f"p{i}", "stage": stage, "ptype": ptype, "episodeId": ep, # noqa: E731
|
||||
"node": "n", "file": f"f{i}", "point_row": 1}
|
||||
target = [mk(i, "bug.run", "T1_repeat", f"t{i}") for i in range(20)]
|
||||
others = [
|
||||
mk(100, "bug.run", "T2_wait", None), # 同 stage 等待点:可进保留集
|
||||
mk(101, "bug.run", "T1_repeat", "real"), # 同 stage 审计过的真失败:可进
|
||||
mk(102, "bug.run", "T1_repeat", "unaudited"), # 没审计过(可能是误报):不进
|
||||
mk(103, "feature.run.first", "T2_wait", None), # 其它 stage:不进
|
||||
]
|
||||
return target, others
|
||||
|
||||
def test_holdout_is_same_stage_and_audited_and_anchor_checked(self):
|
||||
target, others = self._points()
|
||||
cluster = {"stage": "bug.run", "cluster": "x", "replayType": "T1_repeat"}
|
||||
with mock.patch.object(mine, "replay_points", side_effect=lambda c=None: target if c else target + others), \
|
||||
mock.patch.object(mine, "real_episode_ids", return_value={"real"}), \
|
||||
mock.patch.object(mine, "load_rows_for_point", side_effect=lambda p: p["id"]), \
|
||||
mock.patch.object(replay, "candidate_applies", side_effect=lambda rows, ins: rows != "p3"):
|
||||
t, h, seeds, skipped = runner._sample_points(cluster, {"afterLine": "a", "content": "b"})
|
||||
self.assertEqual({p["id"] for p in h}, {"p100", "p101"})
|
||||
self.assertNotIn("p3", {p["id"] for p in t})
|
||||
self.assertEqual(skipped["targetAnchorMissing"], 1)
|
||||
self.assertGreaterEqual(seeds, 2)
|
||||
|
||||
def test_high_error_rate_aborts_without_offline_result(self):
|
||||
target, _ = self._points()
|
||||
cluster = {"stage": "bug.run", "cluster": "x", "replayType": "T1_repeat"}
|
||||
with tempfile.TemporaryDirectory() as d, \
|
||||
mock.patch.object(config, "RUNS", Path(d)), \
|
||||
mock.patch.object(mine, "replay_points", side_effect=lambda c=None: target), \
|
||||
mock.patch.object(mine, "real_episode_ids", return_value=set()), \
|
||||
mock.patch.object(mine, "load_rows_for_point", return_value=[]), \
|
||||
mock.patch.object(replay, "candidate_applies", return_value=True), \
|
||||
mock.patch.object(replay, "prepare"), \
|
||||
mock.patch.object(replay, "run_point", return_value={"first": None, "secs": 0, "error": "Failed to authenticate"}):
|
||||
with self.assertRaises(runner.ReplayAborted):
|
||||
runner.run_replays(cluster, {"afterLine": None, "content": "x"})
|
||||
|
||||
|
||||
class RuntimeProposalTests(unittest.TestCase):
|
||||
KNOBS = {
|
||||
"effort": {"label": "思考强度", "values": ["low", "medium", "high", "xhigh", "max"], "doc": "d"},
|
||||
"ui_settle": {"label": "等界面稳定", "values": ["on"], "doc": "d", "platforms": ["ios"]},
|
||||
}
|
||||
|
||||
def _exp(self, platform, profile):
|
||||
return {"kind": "runtime", "platform": platform, "candidateProfile": json.dumps(profile)}
|
||||
|
||||
def test_skips_tried_baseline_and_off_platform(self):
|
||||
exps = [self._exp("android", {"effort": "medium"}), self._exp("ios", {"ui_settle": "on"})]
|
||||
with mock.patch.object(config, "LAB_BASELINE", {"android": {"effort": "xhigh"}}):
|
||||
# medium 试过、xhigh 是实验机现行 → high;ui_settle 不适用 Android
|
||||
self.assertEqual(runtime.next_candidate("android", self.KNOBS, exps), ("effort", "high"))
|
||||
# iOS:effort 一个都没试过,从 medium 开始
|
||||
self.assertEqual(runtime.next_candidate("ios", self.KNOBS, exps), ("effort", "medium"))
|
||||
|
||||
def test_exhausted_catalog_proposes_nothing(self):
|
||||
exps = [self._exp("ios", {"effort": v}) for v in ["low", "medium", "high", "xhigh", "max"]]
|
||||
exps.append(self._exp("ios", {"ui_settle": "on"}))
|
||||
self.assertIsNone(runtime.next_candidate("ios", self.KNOBS, exps))
|
||||
|
||||
def test_free_lab_nodes_each_get_one(self):
|
||||
state = {"runtimeKnobs": self.KNOBS, "labNodes": LAB_NODES}
|
||||
posted = []
|
||||
with mock.patch.object(runner.api, "post", side_effect=lambda path, body, timeout=120: posted.append(body) or {"experiment": {"id": "x", "title": body["title"], "status": "lab"}}), \
|
||||
mock.patch.object(config, "LAB_BASELINE", {}):
|
||||
got = runner._submit_runtime_experiments(state, [], busy={"android"})
|
||||
self.assertEqual([b["platform"] for b in posted], ["ios"])
|
||||
self.assertEqual(posted[0]["candidateProfile"], {"effort": "medium"})
|
||||
self.assertEqual(got[0]["status"], "lab")
|
||||
|
||||
|
||||
class IosWaitPointTests(unittest.TestCase):
|
||||
def test_catuse_probe_counts_as_ui_wait(self):
|
||||
self.assertTrue(episodes.UI_LIKE.search("sleep 20; catuse apps 2>&1 | head -20"))
|
||||
self.assertTrue(episodes._is_wait("sleep 20; catuse apps 2>&1 | head -20"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,82 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from rsi_loop import config, review_mining
|
||||
|
||||
# 2026-11-02 是周一(北京时间),2026-11-03 是周二
|
||||
MON = 1793548800000 + 3600_000 * 4 # 2026-11-02 04:00 北京
|
||||
TUE = MON + 86_400_000
|
||||
|
||||
|
||||
class LabelTest(unittest.TestCase):
|
||||
def test_batches_posted_and_failures_isolated(self):
|
||||
pending = {"total": 3, "batches": [{"keys": ["a"], "prompt": "p1"}, {"keys": ["b"], "prompt": "p2"}]}
|
||||
answers = [RuntimeError("boom"), {"labels": [{"key": "b", "cat": "ops"}]}]
|
||||
posted = []
|
||||
|
||||
def ask_json(prompt, timeout=0):
|
||||
a = answers.pop(0)
|
||||
if isinstance(a, Exception):
|
||||
raise a
|
||||
return a
|
||||
|
||||
with mock.patch.object(review_mining.api, "get", return_value=pending), \
|
||||
mock.patch.object(review_mining.llm, "ask_json", side_effect=ask_json), \
|
||||
mock.patch.object(review_mining.api, "post", side_effect=lambda path, body, **k: posted.append((path, body)) or {"saved": 1, "dropped": 0}):
|
||||
out = review_mining.label_pending()
|
||||
self.assertEqual(out["failedBatches"], 1)
|
||||
self.assertEqual(out["saved"], 1)
|
||||
self.assertEqual(posted[0][0], "/review-mining/labels")
|
||||
self.assertEqual(posted[0][1]["labeler"], review_mining.LABELER)
|
||||
|
||||
def test_days_query(self):
|
||||
with mock.patch.object(review_mining.api, "get", return_value={"batches": []}) as g:
|
||||
review_mining.label_pending(days=180)
|
||||
g.assert_called_with("/review-mining/pending?days=180", timeout=300)
|
||||
|
||||
|
||||
class ClusterTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.marker = Path(self.tmp.name) / "last"
|
||||
self.patch = mock.patch.object(review_mining, "LAST_CLUSTER", self.marker)
|
||||
self.patch.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.patch.stop()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_only_on_monday_once(self):
|
||||
self.assertFalse(review_mining._cluster_due(TUE))
|
||||
self.assertTrue(review_mining._cluster_due(MON))
|
||||
inp = {"rows": 2, "prompt": "p", "window": {"from": 1, "to": 2}}
|
||||
with mock.patch.object(review_mining.api, "get", return_value=inp), \
|
||||
mock.patch.object(review_mining.llm, "ask_json", return_value={"candidates": [{"family": "x"}]}), \
|
||||
mock.patch.object(review_mining.api, "post", return_value={"saved": 1, "rejected": []}) as post, \
|
||||
mock.patch.object(config, "DRY_RUN", False):
|
||||
out = review_mining.cluster(MON)
|
||||
self.assertEqual(out["saved"], 1)
|
||||
self.assertEqual(post.call_args[0][1]["window"], {"from": 1, "to": 2})
|
||||
self.assertFalse(review_mining._cluster_due(MON)) # 同一天不再跑
|
||||
|
||||
def test_force_and_empty_rows(self):
|
||||
with mock.patch.object(review_mining.api, "get", return_value={"rows": 0, "window": {}}), \
|
||||
mock.patch.object(review_mining.llm, "ask_json") as ask, \
|
||||
mock.patch.object(config, "DRY_RUN", True):
|
||||
out = review_mining.cluster(TUE, force=True)
|
||||
ask.assert_not_called()
|
||||
self.assertEqual(out["rows"], 0)
|
||||
self.assertFalse(self.marker.exists()) # 试运行不落标记
|
||||
|
||||
def test_run_isolates_errors(self):
|
||||
with mock.patch.object(review_mining, "label_pending", side_effect=RuntimeError("down")), \
|
||||
mock.patch.object(review_mining, "cluster", return_value=None):
|
||||
out = review_mining.run(TUE)
|
||||
self.assertIn("error", out["label"])
|
||||
self.assertIn("打标签失败", review_mining.summary_line(out))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user