Initial import of streamline
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import plistlib
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import zipfile
|
||||
|
||||
spec = importlib.util.spec_from_file_location("worker", Path(__file__).with_name("worker.py"))
|
||||
worker = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(worker)
|
||||
|
||||
class WorkerTest(unittest.TestCase):
|
||||
def test_manifest_copy_excludes_accounts_and_tasks(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); source = root/"installed"; source.mkdir()
|
||||
(source/"eddy").write_text("tool")
|
||||
(source/"accounts").write_text("private")
|
||||
(source/"MANIFEST.json").write_text(json.dumps({"kind":"runtime","release":"v2.8","files":[{"path":"eddy","sha256":worker.digest(source/"eddy")}]}))
|
||||
self.assertEqual(worker.copy_runtime(source,root/"out"), "v2.8")
|
||||
self.assertFalse((root/"out/accounts").exists())
|
||||
(source/"eddy").write_text("changed")
|
||||
with self.assertRaises(ValueError):
|
||||
worker.copy_runtime(source,root/"bad")
|
||||
def test_zip_slip_and_symlink_ancestors_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder); z=root/"runtime.zip"
|
||||
for unsafe in ("../outside", "/absolute"):
|
||||
with zipfile.ZipFile(z,"w") as a: a.writestr(unsafe, "oops")
|
||||
with self.assertRaises(ValueError): worker.safe_extract(z,root/"out")
|
||||
with zipfile.ZipFile(z,"w") as a:
|
||||
link=zipfile.ZipInfo("root/link"); link.external_attr=(stat.S_IFLNK|0o777)<<16
|
||||
a.writestr(link,"../../outside")
|
||||
with self.assertRaises(ValueError): worker.safe_extract(z,root/"out")
|
||||
def test_clean_rejects_source_or_commit_drift(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
p=Path(folder)
|
||||
def git(*args): return subprocess.check_output(["git",*args],cwd=p,text=True).strip()
|
||||
git("init","-q")
|
||||
(p/"code").write_text("original")
|
||||
git("add",".")
|
||||
git("-c","user.name=Fixture","-c","user.email=fixture@example.invalid","commit","-qm","fixture")
|
||||
commit=git("rev-parse","HEAD")
|
||||
worker.clean(p,commit)
|
||||
with self.assertRaises(ValueError): worker.clean(p,"a"*40)
|
||||
(p/"new-code").write_text("uncommitted")
|
||||
with self.assertRaises(ValueError): worker.clean(p,commit)
|
||||
(p/"new-code").unlink(); (p/"code").write_text("dirty")
|
||||
with self.assertRaises(subprocess.CalledProcessError): worker.clean(p,commit)
|
||||
def collect_fixture(self, folder, receipt=None, apps=("TikTokM.app",)):
|
||||
root=Path(folder); source=root/"source"; source.mkdir(); (root/"build").mkdir()
|
||||
def git(*args): return subprocess.check_output(["git",*args],cwd=source,text=True).strip()
|
||||
git("init","-q"); (source/"code").write_text("original"); git("add",".")
|
||||
git("-c","user.name=Fixture","-c","user.email=fixture@example.invalid","commit","-qm","fixture")
|
||||
commit=git("rev-parse","HEAD")
|
||||
for name in apps:
|
||||
app=root/"build"/name; app.mkdir()
|
||||
(app/"TikTokM").write_text("binary")
|
||||
(app/"Info.plist").write_bytes(plistlib.dumps({"CFBundleExecutable":"TikTokM"}))
|
||||
shared=root/"shared"; subprocess.check_call(["git","clone","-q",str(source),str(shared)])
|
||||
job={"commit":commit,"repo":str(shared),"recipe":{"product":"tiktok","repoKey":"tiktok-ios","key":"tiktokm","app":"TikTokM.app"}}
|
||||
worker.dump(root/"airbuild-receipt.json", receipt if receipt is not None else {"product":"tiktok","commit":commit})
|
||||
return root, job
|
||||
def test_collect_accepts_exactly_the_declared_app_from_the_pinned_commit(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder)
|
||||
worker.collect(root, job)
|
||||
self.assertEqual(json.loads((root/"built.json").read_text())["app"], str(root/"build/TikTokM.app"))
|
||||
def test_collect_rejects_receipt_for_another_commit_or_product_or_a_failure(self):
|
||||
for receipt in ({"product":"tiktok","commit":"b"*40}, {"product":"notes"}, {"error":"airb sync failed"}):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder, receipt=dict(receipt))
|
||||
if "commit" not in receipt: worker.dump(root/"airbuild-receipt.json", dict(receipt, commit=job["commit"]))
|
||||
with self.assertRaises(ValueError): worker.collect(root, job)
|
||||
def test_collect_rejects_missing_extra_or_wrong_apps(self):
|
||||
for apps in ((), ("TikTokM.app","TikTokNotes.app"), ("TikTokNotes.app",)):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder, apps=apps)
|
||||
with self.assertRaises(ValueError): worker.collect(root, job)
|
||||
def test_collect_rejects_a_dirty_export_checkout(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder)
|
||||
(root/"source/new-file").write_text("agent wrote into the checkout")
|
||||
with self.assertRaises(ValueError): worker.collect(root, job)
|
||||
def test_collect_rejects_a_shared_checkout_the_build_moved_or_dirtied(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder)
|
||||
(Path(job["repo"])/"new-file").write_text("build output in the source tree")
|
||||
with self.assertRaises(ValueError): worker.collect(root, job)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder)
|
||||
shared = Path(job["repo"]); (shared/"code").write_text("moved on")
|
||||
subprocess.check_call(["git","-c","user.name=F","-c","user.email=f@example.invalid","commit","-qam","agent commit"], cwd=shared)
|
||||
with self.assertRaises(ValueError): worker.collect(root, job)
|
||||
def test_checkout_puts_the_pinned_commit_on_the_branch_name(self):
|
||||
# airb sync can only reach a commit that is not on develop through a named branch.
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); repo = root/"repo"; repo.mkdir(); job_root = root/"job"; job_root.mkdir()
|
||||
def git(*args, cwd=repo): return subprocess.check_output(["git",*args],cwd=cwd,text=True).strip()
|
||||
git("init","-q"); git("checkout","-q","-b","feature/x")
|
||||
for n in ("1","2"):
|
||||
(repo/"code").write_text(n); git("add","."); git("-c","user.name=F","-c","user.email=f@example.invalid","commit","-qm",n)
|
||||
older = git("rev-parse","HEAD~1")
|
||||
git("remote","add","origin",str(repo))
|
||||
job = {"repo": str(repo), "commit": older, "start": {"branch": "feature/x", "commit": older, "from": "branch"},
|
||||
"recipe": {"repoKey": "tiktok-ios"}}
|
||||
with patch.object(worker, "runtime", lambda _root: "test"):
|
||||
worker.checkout(job_root, job)
|
||||
source = job_root/"source"
|
||||
self.assertEqual(git("rev-parse","--abbrev-ref","HEAD",cwd=source), "feature/x")
|
||||
self.assertEqual(git("rev-parse","HEAD",cwd=source), older)
|
||||
job["start"]["branch"] = "bad..name"
|
||||
with tempfile.TemporaryDirectory() as again:
|
||||
with patch.object(worker, "runtime", lambda _root: "test"), self.assertRaises(subprocess.CalledProcessError):
|
||||
worker.checkout(Path(again), job)
|
||||
def fake_runtime(self, root, with_extend=True):
|
||||
"""A runtime whose build_app_baseline/devicesim_install record what the worker hands them."""
|
||||
import sys
|
||||
home=root/"runtime"; (home/"vendor/python/bin").mkdir(parents=True)
|
||||
(home/"vendor/python/bin/python3").symlink_to(sys.executable) # package() then runs in this process
|
||||
(home/"portable").mkdir(); (home/"eddyctl").mkdir(); (home/"eddyctl/__init__.py").write_text("")
|
||||
(home/"eddyctl/devicesim_install.py").write_text("ACTIVE=[]\ndef ensure_available(): ACTIVE.append(True)\n")
|
||||
signature="args, extend=None" if with_extend else "args"
|
||||
(home/"portable/build_app_baseline.py").write_text(f"""import json
|
||||
from pathlib import Path
|
||||
CALLS=[]
|
||||
def build({signature}):
|
||||
CALLS.append(dict(vars(args)))
|
||||
stage=Path(args.out)/'stage'/'tiktokm-1.0-100-ab'; stage.mkdir(parents=True)
|
||||
record={{'id':'tiktokm-1.0-100-ab','origin':'central','preparation':{{'files':{{'devicesim.dylib':'d'*64}}}}}}
|
||||
extend(stage, record)
|
||||
(stage/'baseline.json').write_text(json.dumps(record))
|
||||
return {{'baseline':'tiktokm-1.0-100-ab','devicesimSha256':'d'*64,
|
||||
'app':{{'file':'baseline-tiktokm-1.0-100-ab.zip','bytes':1,'sha256':'a'*64}},
|
||||
'source':{{'file':'baseline-tiktokm-1.0-100-ab-source.zip','bytes':2,'sha256':'b'*64}}}}
|
||||
""")
|
||||
worker.dump(root/"toolchain.json", {"eddyVersion":"2026.10.15-eddy-home.12"})
|
||||
worker.dump(root/"built.json", {"app":str(root/"build/TikTokM.app")})
|
||||
import sys as _s
|
||||
saved=list(_s.path)
|
||||
def restore():
|
||||
_s.path[:]=saved
|
||||
for name in [n for n in _s.modules if n in ("eddyctl","portable") or n.startswith(("eddyctl.","portable."))]: del _s.modules[name]
|
||||
self.addCleanup(restore)
|
||||
return home
|
||||
def export_job(self, job):
|
||||
return dict(job, id="exp-1", featureId="f-1", mrUrl="https://example.invalid/mr/1", title="Demo", branch="feature/x",
|
||||
start={"commit":job["commit"],"branch":"feature/x","from":"branch"}, instructions="# relay\n",
|
||||
recipe=dict(job["recipe"], launchDefaults={"arguments":["-skipForceLogin"],"environment":{}}))
|
||||
def test_package_hands_the_build_to_eddy_and_adds_only_the_export_files(self):
|
||||
import sys
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder); job=self.export_job(job); home=self.fake_runtime(root)
|
||||
worker.package(root, job)
|
||||
eddy=sys.modules["portable.build_app_baseline"]
|
||||
self.assertEqual(sys.modules["eddyctl.devicesim_install"].ACTIVE, [True])
|
||||
call=eddy.CALLS[0]
|
||||
# a runtime without vendor/devicesim (after .18): Eddy prepares with the DeviceSim it just activated
|
||||
self.assertEqual((call["app"], call["out"], call["source_repo"], call["source_commit"], call["release"], call["key"], call["tools"]),
|
||||
(root/"build/TikTokM.app", root/"package", root/"source", job["commit"], "2026.10.15-eddy-home.12", "tiktokm", None))
|
||||
self.assertEqual(call["launch_defaults"], {"arguments":["-skipForceLogin"],"environment":{}})
|
||||
stage=root/"package/stage/tiktokm-1.0-100-ab"
|
||||
record=json.loads((stage/"baseline.json").read_text())
|
||||
self.assertEqual((record["origin"], record["devicesimSha256"], record["build"]["receipt"], record["streamline"]["exportId"]),
|
||||
("streamline-task", "d"*64, "BUILD.json", "exp-1"))
|
||||
self.assertEqual(json.loads((stage/"BUILD.json").read_text())["commit"], job["commit"])
|
||||
self.assertEqual((stage/"RELAY.md").read_text(), "# relay\n")
|
||||
self.assertIn("Commit: "+job["commit"], (stage/"INSTALL.md").read_text())
|
||||
result=json.loads((root/"result.json").read_text())
|
||||
self.assertEqual((result["baselineId"], result["eddyVersion"], result["artifacts"]["app"]["file"]),
|
||||
("tiktokm-1.0-100-ab", "2026.10.15-eddy-home.12", "baseline-tiktokm-1.0-100-ab.zip"))
|
||||
def test_package_uses_the_devicesim_an_older_runtime_ships(self):
|
||||
import sys
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder); job=self.export_job(job); home=self.fake_runtime(root)
|
||||
(home/"vendor/devicesim").mkdir()
|
||||
worker.package(root, job)
|
||||
self.assertEqual(sys.modules["portable.build_app_baseline"].CALLS[0]["tools"], home/"vendor/devicesim")
|
||||
def test_package_refuses_a_runtime_without_the_extend_hook(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, job = self.collect_fixture(folder); job=self.export_job(job); self.fake_runtime(root, with_extend=False)
|
||||
with self.assertRaisesRegex(ValueError, "2026.10.15-eddy-home.12"):
|
||||
worker.package(root, job)
|
||||
def test_eddy_and_devicesim_state_stay_inside_the_export(self):
|
||||
env = worker.isolated(Path("/tmp/export-x"), "verify-state")
|
||||
self.assertEqual(env["DEVICESIM_ROOT"], "/tmp/export-x/devicesim")
|
||||
self.assertEqual(env["EDDY_STATE_DIR"], "/tmp/export-x/verify-state")
|
||||
self.assertNotIn("EDDY_DEVICESIM_FEED", env) # newer runtimes download DeviceSim into the job's root
|
||||
self.assertEqual(env["EDDY_CLIENT"], "streamline-worker") # its product usage stays on the node
|
||||
|
||||
if __name__=="__main__": unittest.main()
|
||||
Reference in New Issue
Block a user