Initial import of streamline
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
# Feature → Eddy 导出:卫星机 worker
|
||||
|
||||
`worker.py` 随 satellite bundle 分发,由 `satellite/src/eddy_export.ts` 在每个导出作业目录
|
||||
(`$EDDY_EXPORT_ROOT/<exportId>/`,默认 `~/.streamline/eddy-exports`)里按阶段调用。整体流程、配置和排查见
|
||||
[docs/eddy/feature-export.md](../../docs/eddy/feature-export.md)。
|
||||
|
||||
| 阶段 | 谁做 | 做什么 |
|
||||
| --- | --- | --- |
|
||||
| `checkout` | worker | 拉官方 Eddy runtime(`eddy/LATEST`,校验后只复制分发清单);独立 clone 产品仓,在分支名上检出固定提交(`git checkout -B <分支> <提交>`),源码包从这里出 |
|
||||
| 固定共享检出 | `eddy_export.ts` | 把节点共享检出(`$AI_WORKSPACE/TikTok`)fetch 并清理(与验证任务同一个 `checkoutChangeCommit`),再在分支名上检出固定提交(`pinBranch`;`airb sync` 按分支名取提交) |
|
||||
| 构建 | 智能体 | 按 DeviceSim 的 `Skill(airbuild)` 在共享检出上、用它已绑定的 airspace 远程构建(不 `airb init`、不 `airb release`),把产物拉到 `build/`,写 `airbuild-receipt.json`(`satellite/src/eddy_export_build.ts`) |
|
||||
| `collect` | worker | 只认:独立 clone 和共享检出都在固定提交且无改动、`build/` 里恰好是声明的 `.app`、回执的产品与提交一致 |
|
||||
| `packaging` | worker | 在作业自己的 `DEVICESIM_ROOT` 里装好 DeviceSim(.18 及以前的 runtime 用包里自带的;之后的 runtime 不带,从 DeviceSim 官方渠道下载最新 stable),用它(`cat-supervisor install --skip-tcc`)准备 App,打 App ZIP 与源码 ZIP,写入接力文档和来源 |
|
||||
| `verifying` | worker | 在作业自己的 Eddy state 里安装 ZIP、启动并 smoke |
|
||||
| `cleanup` | worker | 停掉 smoke 启动的 App |
|
||||
|
||||
这里**没有构建脚本**:怎么构建 TikTok / TikTokNotes / Whee 只写在 DeviceSim airbuild skill 的「工程专用」里,
|
||||
和节点上修复/验证任务出包是同一份说明。构建什么(Eddy key、产物名、启动参数)由任务的 product profile 在
|
||||
`shared/src/eddy_export.ts` 的 `EDDY_EXPORT_PRODUCTS` 里查到。
|
||||
|
||||
Eddy 与 cat-supervisor 默认使用机器级目录(`/Users/Shared/DeviceSim`、`~/iosapp`),节点自己的智能体也在用。
|
||||
worker 把 `DEVICESIM_ROOT`、`EDDY_STATE_DIR` 和打包实例目录都指到作业目录里,导出不会切换或触碰它们。
|
||||
|
||||
## 测试
|
||||
|
||||
```bash
|
||||
python3 -m unittest discover -s satellite/eddy-export -v
|
||||
```
|
||||
|
||||
可选真实工具链测试(macOS):用一个最小扁平 Catalyst App 代替构建智能体,经官方 runtime 自带的 cat-supervisor
|
||||
打包并导入 Eddy,不启动 App。`EDDY_EXPORT_TEST_RUNTIME` 指向解压后的官方 runtime(含 `MANIFEST.json` 的目录):
|
||||
|
||||
```bash
|
||||
EDDY_EXPORT_TEST_RUNTIME=/path/to/eddy-runtime python3 -m unittest discover -s satellite/eddy-export -v
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,84 @@
|
||||
"""Opt-in macOS integration with the actual Eddy distribution. Builds a tiny flat Catalyst App in
|
||||
place of the airbuild agent, packages it through the runtime's own cat-supervisor and imports both
|
||||
ZIPs through Eddy. Does not launch any native App.
|
||||
EDDY_EXPORT_TEST_RUNTIME=/path/to/unpacked/official/runtime python3 -m unittest discover -s satellite/eddy-export -v
|
||||
"""
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import zipfile
|
||||
|
||||
spec = importlib.util.spec_from_file_location("worker", Path(__file__).with_name("worker.py"))
|
||||
worker = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(worker)
|
||||
|
||||
@unittest.skipUnless(sys.platform == "darwin" and os.environ.get("EDDY_EXPORT_TEST_RUNTIME"), "requires an explicit Eddy runtime on macOS")
|
||||
class PackagingTest(unittest.TestCase):
|
||||
def test_collect_package_and_import_with_source(self):
|
||||
with tempfile.TemporaryDirectory(prefix="eddy-export-package-") as directory:
|
||||
folder = Path(directory); repo = folder / "repo"; repo.mkdir(); root = folder / "job"; root.mkdir()
|
||||
def git(*args): return subprocess.check_output(["git", *args], cwd=repo, text=True).strip()
|
||||
(repo / "main.c").write_text("int main(void) { return 0; }\n")
|
||||
git("init", "-q"); git("add", "."); git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "fixture")
|
||||
git("remote", "add", "origin", "https://example.invalid/fixture")
|
||||
commit = git("rev-parse", "HEAD")
|
||||
recipe = {"product": "fixture", "repoKey": "tiktok-ios", "key": "export-fixture", "name": "Fixture",
|
||||
"app": "ExportFixture.app", "launchDefaults": {"arguments": ["-skipForceLogin"]}}
|
||||
job = {"id": "packaging-fixture", "featureId": "fixture", "title": "Local packaging fixture", "mrUrl": "https://example.invalid/mr",
|
||||
"commit": commit, "repo": str(repo), "instructions": "# Relay fixture\nRequirements and review history\n", "recipe": recipe}
|
||||
worker.dump(root / "job.json", job)
|
||||
def runtime(target):
|
||||
version = worker.copy_runtime(Path(os.environ["EDDY_EXPORT_TEST_RUNTIME"]), target / "runtime")
|
||||
worker.dump(target / "toolchain.json", {"eddyVersion": version})
|
||||
return version
|
||||
with patch.object(worker, "runtime", runtime):
|
||||
worker.checkout(root, job)
|
||||
# Stand-in for the airbuild agent: a flat Catalyst App in build/ plus its receipt.
|
||||
app = root / "build/ExportFixture.app"; app.mkdir()
|
||||
sdk = subprocess.check_output(["xcrun", "--sdk", "macosx", "--show-sdk-path"], text=True).strip()
|
||||
subprocess.run(["xcrun", "clang", "-target", "arm64-apple-ios14.0-macabi", "-isysroot", sdk, "-Wl,-headerpad,0x1000",
|
||||
str(repo / "main.c"), "-o", str(app / "ExportFixture")], check=True)
|
||||
import plistlib
|
||||
(app / "Info.plist").write_bytes(plistlib.dumps({"CFBundleIdentifier": "com.example.streamline-export-fixture",
|
||||
"CFBundleExecutable": "ExportFixture", "CFBundleName": "ExportFixture", "CFBundlePackageType": "APPL",
|
||||
"CFBundleShortVersionString": "1.0", "CFBundleVersion": "1"}))
|
||||
worker.dump(root / "airbuild-receipt.json", {"product": "fixture", "commit": commit, "remoteApp": "bazel-out/fixture",
|
||||
"commands": ["fixture"], "released": True})
|
||||
worker.collect(root, job)
|
||||
worker.package(root, job)
|
||||
result = json.loads((root / "result.json").read_text())
|
||||
self.assertEqual(result["commit"], commit)
|
||||
for a in result["artifacts"].values():
|
||||
self.assertEqual(worker.digest(root / "artifacts" / a["file"]), a["sha256"])
|
||||
with zipfile.ZipFile(root / "artifacts" / result["artifacts"]["app"]["file"]) as archive:
|
||||
baseline = json.loads(archive.read(result["baselineId"] + "/baseline.json"))
|
||||
self.assertEqual(baseline["source"]["commit"], commit)
|
||||
self.assertEqual(baseline["devicesimSha256"], result["devicesimSha256"])
|
||||
self.assertEqual(baseline["preparation"]["method"], "cat-supervisor install")
|
||||
self.assertEqual(baseline["streamline"]["featureId"], "fixture")
|
||||
self.assertEqual(baseline["streamline"]["relay"], "RELAY.md")
|
||||
self.assertEqual(baseline["build"]["builder"], "airbuild-skill")
|
||||
self.assertEqual(archive.read(result["baselineId"] + "/RELAY.md").decode(), job["instructions"])
|
||||
self.assertEqual((root / "RELAY.md").read_text(), job["instructions"])
|
||||
self.assertEqual(baseline["launchDefaults"]["arguments"], ["-skipForceLogin"])
|
||||
self.assertEqual(json.loads(archive.read(result["baselineId"] + "/BUILD.json"))["commit"], commit)
|
||||
# Packaging must not have switched this machine's DeviceSim or left an instance behind.
|
||||
self.assertTrue((root / "devicesim/current/devicesim.dylib").is_file())
|
||||
home = root / "runtime"
|
||||
env = dict(os.environ, **worker.isolated(root, "import-state"))
|
||||
def eddy(*args):
|
||||
completed = subprocess.run([str(home / "eddy"), *args], cwd=home, env=env, text=True, capture_output=True, timeout=180)
|
||||
self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr)
|
||||
return json.loads(completed.stdout)
|
||||
eddy("setup")
|
||||
receipt = eddy("baseline", "add", str(root / "artifacts" / result["artifacts"]["app"]["file"]), "--wait")
|
||||
self.assertEqual(receipt["installed"], result["baselineId"])
|
||||
manifests = list((root / "import-state").rglob("SOURCE.json")) + list((home / "apps").rglob("SOURCE.json"))
|
||||
self.assertTrue(manifests, "local install must import the sibling source ZIP")
|
||||
self.assertEqual(json.loads(manifests[0].read_text())["commit"], commit)
|
||||
@@ -0,0 +1,194 @@
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import plistlib
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import zipfile
|
||||
|
||||
spec = importlib.util.spec_from_file_location("worker", Path(__file__).with_name("worker.py"))
|
||||
worker = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(worker)
|
||||
|
||||
class WorkerTest(unittest.TestCase):
|
||||
def test_manifest_copy_excludes_accounts_and_tasks(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); source = root/"installed"; source.mkdir()
|
||||
(source/"eddy").write_text("tool")
|
||||
(source/"accounts").write_text("private")
|
||||
(source/"MANIFEST.json").write_text(json.dumps({"kind":"runtime","release":"v2.8","files":[{"path":"eddy","sha256":worker.digest(source/"eddy")}]}))
|
||||
self.assertEqual(worker.copy_runtime(source,root/"out"), "v2.8")
|
||||
self.assertFalse((root/"out/accounts").exists())
|
||||
(source/"eddy").write_text("changed")
|
||||
with self.assertRaises(ValueError):
|
||||
worker.copy_runtime(source,root/"bad")
|
||||
def test_zip_slip_and_symlink_ancestors_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder); z=root/"runtime.zip"
|
||||
for unsafe in ("../outside", "/absolute"):
|
||||
with zipfile.ZipFile(z,"w") as a: a.writestr(unsafe, "oops")
|
||||
with self.assertRaises(ValueError): worker.safe_extract(z,root/"out")
|
||||
with zipfile.ZipFile(z,"w") as a:
|
||||
link=zipfile.ZipInfo("root/link"); link.external_attr=(stat.S_IFLNK|0o777)<<16
|
||||
a.writestr(link,"../../outside")
|
||||
with self.assertRaises(ValueError): worker.safe_extract(z,root/"out")
|
||||
def test_clean_rejects_source_or_commit_drift(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
p=Path(folder)
|
||||
def git(*args): return subprocess.check_output(["git",*args],cwd=p,text=True).strip()
|
||||
git("init","-q")
|
||||
(p/"code").write_text("original")
|
||||
git("add",".")
|
||||
git("-c","user.name=Fixture","-c","user.email=fixture@example.invalid","commit","-qm","fixture")
|
||||
commit=git("rev-parse","HEAD")
|
||||
worker.clean(p,commit)
|
||||
with self.assertRaises(ValueError): worker.clean(p,"a"*40)
|
||||
(p/"new-code").write_text("uncommitted")
|
||||
with self.assertRaises(ValueError): worker.clean(p,commit)
|
||||
(p/"new-code").unlink(); (p/"code").write_text("dirty")
|
||||
with self.assertRaises(subprocess.CalledProcessError): worker.clean(p,commit)
|
||||
def collect_fixture(self, folder, receipt=None, apps=("TikTokM.app",)):
|
||||
root=Path(folder); source=root/"source"; source.mkdir(); (root/"build").mkdir()
|
||||
def git(*args): return subprocess.check_output(["git",*args],cwd=source,text=True).strip()
|
||||
git("init","-q"); (source/"code").write_text("original"); git("add",".")
|
||||
git("-c","user.name=Fixture","-c","user.email=fixture@example.invalid","commit","-qm","fixture")
|
||||
commit=git("rev-parse","HEAD")
|
||||
for name in apps:
|
||||
app=root/"build"/name; app.mkdir()
|
||||
(app/"TikTokM").write_text("binary")
|
||||
(app/"Info.plist").write_bytes(plistlib.dumps({"CFBundleExecutable":"TikTokM"}))
|
||||
shared=root/"shared"; subprocess.check_call(["git","clone","-q",str(source),str(shared)])
|
||||
job={"commit":commit,"repo":str(shared),"recipe":{"product":"tiktok","repoKey":"tiktok-ios","key":"tiktokm","app":"TikTokM.app"}}
|
||||
worker.dump(root/"airbuild-receipt.json", receipt if receipt is not None else {"product":"tiktok","commit":commit})
|
||||
return root, job
|
||||
def test_collect_accepts_exactly_the_declared_app_from_the_pinned_commit(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder)
|
||||
worker.collect(root, job)
|
||||
self.assertEqual(json.loads((root/"built.json").read_text())["app"], str(root/"build/TikTokM.app"))
|
||||
def test_collect_rejects_receipt_for_another_commit_or_product_or_a_failure(self):
|
||||
for receipt in ({"product":"tiktok","commit":"b"*40}, {"product":"notes"}, {"error":"airb sync failed"}):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder, receipt=dict(receipt))
|
||||
if "commit" not in receipt: worker.dump(root/"airbuild-receipt.json", dict(receipt, commit=job["commit"]))
|
||||
with self.assertRaises(ValueError): worker.collect(root, job)
|
||||
def test_collect_rejects_missing_extra_or_wrong_apps(self):
|
||||
for apps in ((), ("TikTokM.app","TikTokNotes.app"), ("TikTokNotes.app",)):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder, apps=apps)
|
||||
with self.assertRaises(ValueError): worker.collect(root, job)
|
||||
def test_collect_rejects_a_dirty_export_checkout(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder)
|
||||
(root/"source/new-file").write_text("agent wrote into the checkout")
|
||||
with self.assertRaises(ValueError): worker.collect(root, job)
|
||||
def test_collect_rejects_a_shared_checkout_the_build_moved_or_dirtied(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder)
|
||||
(Path(job["repo"])/"new-file").write_text("build output in the source tree")
|
||||
with self.assertRaises(ValueError): worker.collect(root, job)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder)
|
||||
shared = Path(job["repo"]); (shared/"code").write_text("moved on")
|
||||
subprocess.check_call(["git","-c","user.name=F","-c","user.email=f@example.invalid","commit","-qam","agent commit"], cwd=shared)
|
||||
with self.assertRaises(ValueError): worker.collect(root, job)
|
||||
def test_checkout_puts_the_pinned_commit_on_the_branch_name(self):
|
||||
# airb sync can only reach a commit that is not on develop through a named branch.
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); repo = root/"repo"; repo.mkdir(); job_root = root/"job"; job_root.mkdir()
|
||||
def git(*args, cwd=repo): return subprocess.check_output(["git",*args],cwd=cwd,text=True).strip()
|
||||
git("init","-q"); git("checkout","-q","-b","feature/x")
|
||||
for n in ("1","2"):
|
||||
(repo/"code").write_text(n); git("add","."); git("-c","user.name=F","-c","user.email=f@example.invalid","commit","-qm",n)
|
||||
older = git("rev-parse","HEAD~1")
|
||||
git("remote","add","origin",str(repo))
|
||||
job = {"repo": str(repo), "commit": older, "start": {"branch": "feature/x", "commit": older, "from": "branch"},
|
||||
"recipe": {"repoKey": "tiktok-ios"}}
|
||||
with patch.object(worker, "runtime", lambda _root: "test"):
|
||||
worker.checkout(job_root, job)
|
||||
source = job_root/"source"
|
||||
self.assertEqual(git("rev-parse","--abbrev-ref","HEAD",cwd=source), "feature/x")
|
||||
self.assertEqual(git("rev-parse","HEAD",cwd=source), older)
|
||||
job["start"]["branch"] = "bad..name"
|
||||
with tempfile.TemporaryDirectory() as again:
|
||||
with patch.object(worker, "runtime", lambda _root: "test"), self.assertRaises(subprocess.CalledProcessError):
|
||||
worker.checkout(Path(again), job)
|
||||
def fake_runtime(self, root, with_extend=True):
|
||||
"""A runtime whose build_app_baseline/devicesim_install record what the worker hands them."""
|
||||
import sys
|
||||
home=root/"runtime"; (home/"vendor/python/bin").mkdir(parents=True)
|
||||
(home/"vendor/python/bin/python3").symlink_to(sys.executable) # package() then runs in this process
|
||||
(home/"portable").mkdir(); (home/"eddyctl").mkdir(); (home/"eddyctl/__init__.py").write_text("")
|
||||
(home/"eddyctl/devicesim_install.py").write_text("ACTIVE=[]\ndef ensure_available(): ACTIVE.append(True)\n")
|
||||
signature="args, extend=None" if with_extend else "args"
|
||||
(home/"portable/build_app_baseline.py").write_text(f"""import json
|
||||
from pathlib import Path
|
||||
CALLS=[]
|
||||
def build({signature}):
|
||||
CALLS.append(dict(vars(args)))
|
||||
stage=Path(args.out)/'stage'/'tiktokm-1.0-100-ab'; stage.mkdir(parents=True)
|
||||
record={{'id':'tiktokm-1.0-100-ab','origin':'central','preparation':{{'files':{{'devicesim.dylib':'d'*64}}}}}}
|
||||
extend(stage, record)
|
||||
(stage/'baseline.json').write_text(json.dumps(record))
|
||||
return {{'baseline':'tiktokm-1.0-100-ab','devicesimSha256':'d'*64,
|
||||
'app':{{'file':'baseline-tiktokm-1.0-100-ab.zip','bytes':1,'sha256':'a'*64}},
|
||||
'source':{{'file':'baseline-tiktokm-1.0-100-ab-source.zip','bytes':2,'sha256':'b'*64}}}}
|
||||
""")
|
||||
worker.dump(root/"toolchain.json", {"eddyVersion":"2026.10.15-eddy-home.12"})
|
||||
worker.dump(root/"built.json", {"app":str(root/"build/TikTokM.app")})
|
||||
import sys as _s
|
||||
saved=list(_s.path)
|
||||
def restore():
|
||||
_s.path[:]=saved
|
||||
for name in [n for n in _s.modules if n in ("eddyctl","portable") or n.startswith(("eddyctl.","portable."))]: del _s.modules[name]
|
||||
self.addCleanup(restore)
|
||||
return home
|
||||
def export_job(self, job):
|
||||
return dict(job, id="exp-1", featureId="f-1", mrUrl="https://example.invalid/mr/1", title="Demo", branch="feature/x",
|
||||
start={"commit":job["commit"],"branch":"feature/x","from":"branch"}, instructions="# relay\n",
|
||||
recipe=dict(job["recipe"], launchDefaults={"arguments":["-skipForceLogin"],"environment":{}}))
|
||||
def test_package_hands_the_build_to_eddy_and_adds_only_the_export_files(self):
|
||||
import sys
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder); job=self.export_job(job); home=self.fake_runtime(root)
|
||||
worker.package(root, job)
|
||||
eddy=sys.modules["portable.build_app_baseline"]
|
||||
self.assertEqual(sys.modules["eddyctl.devicesim_install"].ACTIVE, [True])
|
||||
call=eddy.CALLS[0]
|
||||
# a runtime without vendor/devicesim (after .18): Eddy prepares with the DeviceSim it just activated
|
||||
self.assertEqual((call["app"], call["out"], call["source_repo"], call["source_commit"], call["release"], call["key"], call["tools"]),
|
||||
(root/"build/TikTokM.app", root/"package", root/"source", job["commit"], "2026.10.15-eddy-home.12", "tiktokm", None))
|
||||
self.assertEqual(call["launch_defaults"], {"arguments":["-skipForceLogin"],"environment":{}})
|
||||
stage=root/"package/stage/tiktokm-1.0-100-ab"
|
||||
record=json.loads((stage/"baseline.json").read_text())
|
||||
self.assertEqual((record["origin"], record["devicesimSha256"], record["build"]["receipt"], record["streamline"]["exportId"]),
|
||||
("streamline-task", "d"*64, "BUILD.json", "exp-1"))
|
||||
self.assertEqual(json.loads((stage/"BUILD.json").read_text())["commit"], job["commit"])
|
||||
self.assertEqual((stage/"RELAY.md").read_text(), "# relay\n")
|
||||
self.assertIn("Commit: "+job["commit"], (stage/"INSTALL.md").read_text())
|
||||
result=json.loads((root/"result.json").read_text())
|
||||
self.assertEqual((result["baselineId"], result["eddyVersion"], result["artifacts"]["app"]["file"]),
|
||||
("tiktokm-1.0-100-ab", "2026.10.15-eddy-home.12", "baseline-tiktokm-1.0-100-ab.zip"))
|
||||
def test_package_uses_the_devicesim_an_older_runtime_ships(self):
|
||||
import sys
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder); job=self.export_job(job); home=self.fake_runtime(root)
|
||||
(home/"vendor/devicesim").mkdir()
|
||||
worker.package(root, job)
|
||||
self.assertEqual(sys.modules["portable.build_app_baseline"].CALLS[0]["tools"], home/"vendor/devicesim")
|
||||
def test_package_refuses_a_runtime_without_the_extend_hook(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder); job=self.export_job(job); self.fake_runtime(root, with_extend=False)
|
||||
with self.assertRaisesRegex(ValueError, "2026.10.15-eddy-home.12"):
|
||||
worker.package(root, job)
|
||||
def test_eddy_and_devicesim_state_stay_inside_the_export(self):
|
||||
env = worker.isolated(Path("/tmp/export-x"), "verify-state")
|
||||
self.assertEqual(env["DEVICESIM_ROOT"], "/tmp/export-x/devicesim")
|
||||
self.assertEqual(env["EDDY_STATE_DIR"], "/tmp/export-x/verify-state")
|
||||
self.assertNotIn("EDDY_DEVICESIM_FEED", env) # newer runtimes download DeviceSim into the job's root
|
||||
self.assertEqual(env["EDDY_CLIENT"], "streamline-worker") # its product usage stays on the node
|
||||
|
||||
if __name__=="__main__": unittest.main()
|
||||
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Per-export checkout/collect/package/smoke worker. The build between checkout and collect is an
|
||||
agent following DeviceSim's airbuild skill (satellite/src/eddy_export_build.ts); this file only
|
||||
checks what it hands back. No global Eddy installation, DeviceSim selection or product checkout
|
||||
mutations."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import plistlib
|
||||
import shutil
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
import zipfile
|
||||
|
||||
OFFICIAL = "https://voffline.byted.org/download/tos/schedule/tiktok-social-bot-config/eddy/"
|
||||
|
||||
def digest(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
def dump(path, value):
|
||||
Path(path).write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
|
||||
|
||||
def run(args, cwd=None, env=None, capture=False, timeout=7200):
|
||||
result = subprocess.run(list(map(str, args)), cwd=cwd, env=env, check=True, timeout=timeout,
|
||||
stdout=subprocess.PIPE if capture else None, text=capture)
|
||||
return result.stdout.strip() if capture else None
|
||||
|
||||
def inside(root, path):
|
||||
root, path = Path(root).resolve(), Path(path).resolve()
|
||||
if path == root or root not in path.parents:
|
||||
raise ValueError("Path escapes owned export directory: " + str(path))
|
||||
return path
|
||||
|
||||
def read_url(url):
|
||||
if not url.startswith(OFFICIAL) or any(c in url for c in ["\n", "\r", "?", "#"]):
|
||||
raise ValueError("Expected an official Eddy distribution URL")
|
||||
with urllib.request.urlopen(url, timeout=90) as response:
|
||||
if not response.geturl().startswith(OFFICIAL):
|
||||
raise ValueError("Unexpected distribution redirect")
|
||||
return response.read(1024 * 1024)
|
||||
|
||||
def safe_extract(archive, destination):
|
||||
"""Validate paths and symlink targets before preserving the runtime's executable bits."""
|
||||
root = Path(destination)
|
||||
with zipfile.ZipFile(archive) as z:
|
||||
entries = [i for i in z.infolist() if not i.filename.startswith("__MACOSX/")]
|
||||
links = {i.filename.rstrip("/") for i in entries if stat.S_ISLNK(i.external_attr >> 16)}
|
||||
for i in entries:
|
||||
rel = Path(i.filename)
|
||||
inside(root, root / rel)
|
||||
if rel.is_absolute() or ".." in rel.parts or any(str(p) in links for p in rel.parents):
|
||||
raise ValueError("Unsafe ZIP entry")
|
||||
for i in entries:
|
||||
dest = root / i.filename
|
||||
if i.is_dir():
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
continue
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
mode = i.external_attr >> 16
|
||||
if stat.S_ISLNK(mode):
|
||||
target = z.read(i).decode()
|
||||
inside(root, dest.parent / target)
|
||||
dest.symlink_to(target)
|
||||
else:
|
||||
with z.open(i) as source, dest.open("wb") as output:
|
||||
shutil.copyfileobj(source, output)
|
||||
dest.chmod(stat.S_IMODE(mode) or 0o644)
|
||||
|
||||
def copy_runtime(source, dest):
|
||||
"""Copy manifest-listed distribution files only, excluding installed apps, accounts and tasks."""
|
||||
manifest = json.loads((source / "MANIFEST.json").read_text())
|
||||
if manifest.get("kind") != "runtime":
|
||||
raise ValueError("Expected standalone Eddy runtime distribution")
|
||||
dest.mkdir(parents=True)
|
||||
for entry in manifest["files"]:
|
||||
p = source / entry["path"]
|
||||
q = inside(dest, dest / entry["path"])
|
||||
inside(source, p)
|
||||
q.parent.mkdir(parents=True, exist_ok=True)
|
||||
if "symlink" in entry:
|
||||
if not p.is_symlink() or os.readlink(p) != entry["symlink"]:
|
||||
raise ValueError("Runtime symlink mismatch")
|
||||
inside(dest, q.parent / entry["symlink"])
|
||||
q.symlink_to(entry["symlink"])
|
||||
elif "sha256" in entry:
|
||||
if digest(p) != entry["sha256"]:
|
||||
raise ValueError("Runtime file digest mismatch: " + entry["path"])
|
||||
shutil.copy2(p, q)
|
||||
shutil.copy2(source / "MANIFEST.json", dest / "MANIFEST.json")
|
||||
return manifest["release"]
|
||||
|
||||
def runtime(root):
|
||||
url = read_url(OFFICIAL + "LATEST").decode().strip()
|
||||
metadata = json.loads(read_url(url.rsplit("/", 1)[0] + "/release.json"))
|
||||
artifact = metadata["artifact"]
|
||||
if (artifact.get("url") != url or not url.startswith(OFFICIAL) or artifact.get("kind") != "runtime"
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", artifact.get("sha256", ""))
|
||||
or type(artifact.get("bytes")) is not int or not 0 < artifact["bytes"] <= 1024 ** 3):
|
||||
raise ValueError("Eddy release pointer mismatch")
|
||||
cache = Path.home() / ".cache/streamline/eddy" / artifact["sha256"]
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
archive = cache / "runtime.zip"
|
||||
if not archive.is_file() or archive.stat().st_size != artifact["bytes"] or digest(archive) != artifact["sha256"]:
|
||||
temporary = cache / "runtime.pending"
|
||||
with urllib.request.urlopen(url, timeout=90) as response, temporary.open("wb") as output:
|
||||
if not response.geturl().startswith(OFFICIAL):
|
||||
raise ValueError("Unexpected runtime redirect")
|
||||
shutil.copyfileobj(response, output, 1024 * 1024)
|
||||
if temporary.stat().st_size != artifact["bytes"] or digest(temporary) != artifact["sha256"]:
|
||||
raise ValueError("Runtime ZIP digest mismatch")
|
||||
temporary.replace(archive)
|
||||
unpacked = cache / "unpacked"
|
||||
if not (cache / "complete").exists():
|
||||
if unpacked.exists():
|
||||
shutil.rmtree(unpacked)
|
||||
safe_extract(archive, unpacked)
|
||||
(cache / "complete").touch()
|
||||
candidates = list(unpacked.glob("*/MANIFEST.json"))
|
||||
if len(candidates) != 1:
|
||||
raise ValueError("Invalid Eddy runtime ZIP")
|
||||
source = candidates[0].parent
|
||||
version = copy_runtime(source, root / "runtime")
|
||||
dump(root / "toolchain.json", {"eddyVersion": version})
|
||||
return version
|
||||
|
||||
def clean(source, commit):
|
||||
if run(["git", "rev-parse", "HEAD"], cwd=source, capture=True) != commit:
|
||||
raise ValueError("Product checkout changed commit during export")
|
||||
run(["git", "-c", "core.fsmonitor=false", "diff", "--exit-code", "HEAD", "--"], cwd=source)
|
||||
# New source files would not be part of the commit/source archive.
|
||||
untracked = run(["git", "ls-files", "--others", "--exclude-standard"], cwd=source, capture=True)
|
||||
if untracked:
|
||||
raise ValueError("Build left untracked source files; use ignored/generated build paths")
|
||||
|
||||
def checkout(root, job):
|
||||
if job.get("platform", "ios") != "ios" or job["recipe"].get("repoKey") != "tiktok-ios":
|
||||
raise ValueError("Eddy relay currently supports iOS TikTok-family jobs only")
|
||||
runtime(root)
|
||||
source = root / "source"
|
||||
# A clone owns its refs/worktree. Fetching cannot switch the implementation checkout.
|
||||
run(["git", "clone", "--shared", "--no-checkout", job["repo"], source])
|
||||
remote = run(["git", "remote", "get-url", "origin"], cwd=job["repo"], capture=True)
|
||||
run(["git", "remote", "set-url", "origin", remote], cwd=source)
|
||||
try:
|
||||
run(["git", "cat-file", "-e", job["commit"] + "^{commit}"], cwd=source)
|
||||
except subprocess.CalledProcessError:
|
||||
run(["git", "fetch", "--no-tags", "origin", job["commit"]], cwd=source)
|
||||
# The pinned commit on the branch's own name. The App is built in the node's shared checkout
|
||||
# (eddy_export.ts pins it); this clone is what the source archive and relay files are cut from.
|
||||
branch = job["start"]["branch"]
|
||||
run(["git", "check-ref-format", "--branch", branch], cwd=source)
|
||||
run(["git", "checkout", "-B", branch, job["commit"]], cwd=source)
|
||||
(root / "build").mkdir()
|
||||
clean(source, job["commit"])
|
||||
|
||||
def collect(root, job):
|
||||
"""Accept what the build agent handed back: exactly the declared .app, and a receipt that names
|
||||
the pinned commit. The receipt is the agent's declaration, recorded as BUILD.json; the source
|
||||
archive is cut from the pinned commit regardless."""
|
||||
clean(root / "source", job["commit"])
|
||||
# The build ran in the node's shared checkout: it must still be the pinned commit, unmodified,
|
||||
# or the App would not match the source archive.
|
||||
clean(Path(job["repo"]), job["commit"])
|
||||
recipe = job["recipe"]
|
||||
receipt_path = root / "airbuild-receipt.json"
|
||||
if not receipt_path.is_file():
|
||||
raise ValueError("Build agent wrote no receipt")
|
||||
receipt = json.loads(receipt_path.read_text())
|
||||
if receipt.get("error"):
|
||||
raise ValueError("Build agent reported a failure: " + str(receipt["error"])[:300])
|
||||
if receipt.get("product") != recipe["product"] or receipt.get("commit") != job["commit"]:
|
||||
raise ValueError("Build receipt does not name this export's product and pinned commit")
|
||||
apps = sorted(p.name for p in (root / "build").iterdir() if p.suffix == ".app")
|
||||
if apps != [recipe["app"]]:
|
||||
raise ValueError("Expected exactly " + recipe["app"] + " in the build directory, found " + repr(apps))
|
||||
app = root / "build" / recipe["app"]
|
||||
if app.is_symlink() or not app.is_dir() or not (app / "Info.plist").is_file():
|
||||
raise ValueError("Build did not produce a flat " + recipe["app"])
|
||||
executable = plistlib.loads((app / "Info.plist").read_bytes()).get("CFBundleExecutable", "")
|
||||
if not executable or Path(executable).name != executable or not (app / executable).is_file():
|
||||
raise ValueError("Built App has no valid CFBundleExecutable")
|
||||
dump(root / "built.json", {"app": str(app), "commit": job["commit"]})
|
||||
|
||||
def isolated(root, state):
|
||||
"""Eddy and cat-supervisor default to machine-wide locations (/Users/Shared/DeviceSim,
|
||||
~/iosapp) that this node's own agents use. An export must never switch or touch those.
|
||||
DeviceSim's own release channel stays reachable: Eddy runtimes after 2026.10.23-eddy-home.18 carry no
|
||||
DeviceSim and install the latest official stable into the job's DEVICESIM_ROOT.
|
||||
EDDY_CLIENT marks its product usage `satellite`, which an Eddy runtime never uploads."""
|
||||
return {"EDDY_STATE_DIR": str(root / state), "DEVICESIM_ROOT": str(root / "devicesim"),
|
||||
"EDDY_STREAMLINE_URL": "http://127.0.0.1:1", "EDDY_STREAMLINE_TOKEN": "", "EDDY_CLIENT": "streamline-worker"}
|
||||
|
||||
# Eddy's build_app_baseline writes its work, stage and artifacts under one directory.
|
||||
PACKAGE = "package"
|
||||
|
||||
def package(root, job):
|
||||
home = root / "runtime"
|
||||
# Run packaging imports with Eddy's bundled dependencies, not host site-packages.
|
||||
python = home / "vendor/python/bin/python3"
|
||||
if Path(sys.executable).resolve() != python.resolve():
|
||||
env = dict(os.environ, PYTHONPATH=str(home / "vendor/site-packages"), **isolated(root, "package-state"))
|
||||
run([python, __file__, "packaging", root], cwd=root, env=env)
|
||||
return
|
||||
sys.path[:0] = [str(home), str(home / "runtime/supervisor")]
|
||||
from portable import build_app_baseline
|
||||
from eddyctl import devicesim_install
|
||||
import argparse, inspect
|
||||
version = json.loads((root / "toolchain.json").read_text())["eddyVersion"]
|
||||
if "extend" not in inspect.signature(build_app_baseline.build).parameters:
|
||||
raise ValueError("Eddy runtime " + version + " cannot add export files to a baseline; needs 2026.10.15-eddy-home.12 or newer")
|
||||
source = root / "source"
|
||||
clean(source, job["commit"])
|
||||
recipe = job["recipe"]
|
||||
app = Path(json.loads((root / "built.json").read_text())["app"])
|
||||
|
||||
def extend(stage, record):
|
||||
"""What an export adds to Eddy's baseline: task provenance, the build receipt, relay and install notes."""
|
||||
record["origin"] = "streamline-task"
|
||||
record["streamline"] = {"featureId": job["featureId"], "mrUrl": job["mrUrl"], "commit": job["commit"], "exportId": job["id"],
|
||||
"branch": job.get("branch"), "start": job.get("start"), "relay": "RELAY.md"}
|
||||
record["devicesimSha256"] = record["preparation"]["files"]["devicesim.dylib"]
|
||||
record["build"] = {"builder": "airbuild-skill", "receipt": "BUILD.json"}
|
||||
shutil.copy2(root / "airbuild-receipt.json", stage / "BUILD.json")
|
||||
relay = job.get("instructions") or ("# " + job["title"] + "\n")
|
||||
(stage / "RELAY.md").write_text(relay)
|
||||
(root / "RELAY.md").write_text(relay)
|
||||
(stage / "INSTALL.md").write_text(
|
||||
"# " + job["title"] + "\n\nCommit: " + job["commit"] + "\nMR: " + job["mrUrl"] +
|
||||
"\n\nDownload the App ZIP and the matching source ZIP into the same directory. "
|
||||
"In Eddy > Baselines, enter the App ZIP path and install. Then create a task using this baseline. "
|
||||
"Give RELAY.md to your local agent to continue with the requirements, progress and feedback history. "
|
||||
"Sign in with your own account. Feature launch defaults are stored in baseline.json.\n"
|
||||
"BUILD.json is the build agent's declaration of the remote Airbuild it ran. "
|
||||
"The source ZIP is the original committed source.\n")
|
||||
|
||||
# Eddy's own baseline producer: cat-supervisor from the job's DeviceSim converts, injects and signs, so
|
||||
# activate a DeviceSim under the job-local DEVICESIM_ROOT first: the one a runtime up to .18 ships in
|
||||
# vendor/devicesim, else the latest official stable (newer runtimes carry none; Eddy then uses the active one).
|
||||
devicesim_install.ensure_available()
|
||||
vendored = home / "vendor/devicesim"
|
||||
receipt = build_app_baseline.build(argparse.Namespace(
|
||||
app=app, out=root / PACKAGE, source_repo=source, source_commit=job["commit"], release=version,
|
||||
key=recipe["key"], tools=vendored if vendored.is_dir() else None, launch_defaults=recipe.get("launchDefaults")), extend=extend)
|
||||
clean(source, job["commit"])
|
||||
dump(root / "result.json", {"baselineId": receipt["baseline"], "commit": job["commit"], "eddyVersion": version,
|
||||
"devicesimSha256": receipt["devicesimSha256"], "artifacts": {"app": receipt["app"], "source": receipt["source"]}})
|
||||
|
||||
def verify(root, job):
|
||||
result = json.loads((root / "result.json").read_text())
|
||||
artifacts = root / PACKAGE / "artifacts"
|
||||
for artifact in result["artifacts"].values():
|
||||
p = artifacts / artifact["file"]
|
||||
if p.stat().st_size != artifact["bytes"] or digest(p) != artifact["sha256"]:
|
||||
raise ValueError("Package digest mismatch")
|
||||
with zipfile.ZipFile(artifacts / result["artifacts"]["source"]["file"]) as z:
|
||||
source = json.loads(z.read(result["baselineId"] + "-source/SOURCE.json"))
|
||||
if source["commit"] != job["commit"]:
|
||||
raise ValueError("Source archive commit mismatch")
|
||||
home = root / "runtime"
|
||||
env = dict(os.environ, **isolated(root, "verify-state"))
|
||||
def eddy(args):
|
||||
return json.loads(run([home / "eddy", *args], cwd=home, env=env, capture=True, timeout=600))
|
||||
started = False
|
||||
try:
|
||||
eddy(["setup"])
|
||||
eddy(["baseline", "add", str(artifacts / result["artifacts"]["app"]["file"]), "--no-source", "--wait"])
|
||||
eddy(["select", job["recipe"]["key"] + "@" + result["baselineId"]])
|
||||
# The DeviceSim packaging activated under the job-local DEVICESIM_ROOT.
|
||||
eddy(["devicesim", "select", "--baseline", result["baselineId"], "--version", "installed"])
|
||||
started = True
|
||||
eddy(["up", job["recipe"]["key"], "--wait"])
|
||||
smoke = eddy(["smoke", job["recipe"]["key"]])
|
||||
state = smoke.get("status", {})
|
||||
if (not smoke.get("passed") or state.get("state") != "active" or state.get("editCount") != 0
|
||||
or state.get("engineAttached") is not False or state.get("captureMode") != "on-demand"
|
||||
or state.get("recording") is not False):
|
||||
raise ValueError("Native smoke did not prove an active clean baseline")
|
||||
image = Path(smoke["screenshot"])
|
||||
if not image.is_file() or image.read_bytes()[:8] != b"\x89PNG\r\n\x1a\n":
|
||||
raise ValueError("Native screenshot is missing")
|
||||
shutil.copy2(image, root / "smoke.png")
|
||||
dump(root / "verification.json", smoke)
|
||||
finally:
|
||||
if started:
|
||||
eddy(["down", job["recipe"]["key"]])
|
||||
|
||||
def cleanup(root, job):
|
||||
if not (root / "verify-state/installation.json").is_file():
|
||||
return
|
||||
env = dict(os.environ, **isolated(root, "verify-state"))
|
||||
run([root / "runtime/eddy", "down", job["recipe"]["key"]], cwd=root / "runtime", env=env, timeout=45)
|
||||
# The smoke screenshot spawned catuse's machine-wide capture daemon from the job's DeviceSim;
|
||||
# Eddy stops that one (the next catuse call respawns it from the node's own DeviceSim).
|
||||
run([root / "runtime/vendor/python/bin/python3", "-c", "from eddyctl import devicesim_install as d; d.stop_capture_daemon()"],
|
||||
cwd=root / "runtime", env=env, timeout=30)
|
||||
|
||||
if __name__ == "__main__":
|
||||
def stop(_signum, _frame):
|
||||
raise InterruptedError("Export interrupted")
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
stage, directory = sys.argv[1:]
|
||||
root = Path(directory).resolve()
|
||||
job = json.loads((root / "job.json").read_text())
|
||||
{"checkout": checkout, "collect": collect, "packaging": package, "verifying": verify, "cleanup": cleanup}[stage](root, job)
|
||||
Reference in New Issue
Block a user