Files
streamline/satellite/eddy-export/worker.py
T
2026-11-16 17:26:44 -08:00

314 lines
17 KiB
Python

#!/usr/bin/env python3
"""Per-export checkout/collect/package/smoke worker. The build between checkout and collect is an
agent following DeviceSim's airbuild skill (satellite/src/eddy_export_build.ts); this file only
checks what it hands back. No global Eddy installation, DeviceSim selection or product checkout
mutations."""
import hashlib
import json
import os
import re
from pathlib import Path
import plistlib
import shutil
import signal
import stat
import subprocess
import sys
import urllib.request
import zipfile
OFFICIAL = "https://voffline.byted.org/download/tos/schedule/tiktok-social-bot-config/eddy/"
def digest(path):
h = hashlib.sha256()
with Path(path).open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
h.update(chunk)
return h.hexdigest()
def dump(path, value):
Path(path).write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
def run(args, cwd=None, env=None, capture=False, timeout=7200):
result = subprocess.run(list(map(str, args)), cwd=cwd, env=env, check=True, timeout=timeout,
stdout=subprocess.PIPE if capture else None, text=capture)
return result.stdout.strip() if capture else None
def inside(root, path):
root, path = Path(root).resolve(), Path(path).resolve()
if path == root or root not in path.parents:
raise ValueError("Path escapes owned export directory: " + str(path))
return path
def read_url(url):
if not url.startswith(OFFICIAL) or any(c in url for c in ["\n", "\r", "?", "#"]):
raise ValueError("Expected an official Eddy distribution URL")
with urllib.request.urlopen(url, timeout=90) as response:
if not response.geturl().startswith(OFFICIAL):
raise ValueError("Unexpected distribution redirect")
return response.read(1024 * 1024)
def safe_extract(archive, destination):
"""Validate paths and symlink targets before preserving the runtime's executable bits."""
root = Path(destination)
with zipfile.ZipFile(archive) as z:
entries = [i for i in z.infolist() if not i.filename.startswith("__MACOSX/")]
links = {i.filename.rstrip("/") for i in entries if stat.S_ISLNK(i.external_attr >> 16)}
for i in entries:
rel = Path(i.filename)
inside(root, root / rel)
if rel.is_absolute() or ".." in rel.parts or any(str(p) in links for p in rel.parents):
raise ValueError("Unsafe ZIP entry")
for i in entries:
dest = root / i.filename
if i.is_dir():
dest.mkdir(parents=True, exist_ok=True)
continue
dest.parent.mkdir(parents=True, exist_ok=True)
mode = i.external_attr >> 16
if stat.S_ISLNK(mode):
target = z.read(i).decode()
inside(root, dest.parent / target)
dest.symlink_to(target)
else:
with z.open(i) as source, dest.open("wb") as output:
shutil.copyfileobj(source, output)
dest.chmod(stat.S_IMODE(mode) or 0o644)
def copy_runtime(source, dest):
"""Copy manifest-listed distribution files only, excluding installed apps, accounts and tasks."""
manifest = json.loads((source / "MANIFEST.json").read_text())
if manifest.get("kind") != "runtime":
raise ValueError("Expected standalone Eddy runtime distribution")
dest.mkdir(parents=True)
for entry in manifest["files"]:
p = source / entry["path"]
q = inside(dest, dest / entry["path"])
inside(source, p)
q.parent.mkdir(parents=True, exist_ok=True)
if "symlink" in entry:
if not p.is_symlink() or os.readlink(p) != entry["symlink"]:
raise ValueError("Runtime symlink mismatch")
inside(dest, q.parent / entry["symlink"])
q.symlink_to(entry["symlink"])
elif "sha256" in entry:
if digest(p) != entry["sha256"]:
raise ValueError("Runtime file digest mismatch: " + entry["path"])
shutil.copy2(p, q)
shutil.copy2(source / "MANIFEST.json", dest / "MANIFEST.json")
return manifest["release"]
def runtime(root):
url = read_url(OFFICIAL + "LATEST").decode().strip()
metadata = json.loads(read_url(url.rsplit("/", 1)[0] + "/release.json"))
artifact = metadata["artifact"]
if (artifact.get("url") != url or not url.startswith(OFFICIAL) or artifact.get("kind") != "runtime"
or not re.fullmatch(r"[0-9a-f]{64}", artifact.get("sha256", ""))
or type(artifact.get("bytes")) is not int or not 0 < artifact["bytes"] <= 1024 ** 3):
raise ValueError("Eddy release pointer mismatch")
cache = Path.home() / ".cache/streamline/eddy" / artifact["sha256"]
cache.mkdir(parents=True, exist_ok=True)
archive = cache / "runtime.zip"
if not archive.is_file() or archive.stat().st_size != artifact["bytes"] or digest(archive) != artifact["sha256"]:
temporary = cache / "runtime.pending"
with urllib.request.urlopen(url, timeout=90) as response, temporary.open("wb") as output:
if not response.geturl().startswith(OFFICIAL):
raise ValueError("Unexpected runtime redirect")
shutil.copyfileobj(response, output, 1024 * 1024)
if temporary.stat().st_size != artifact["bytes"] or digest(temporary) != artifact["sha256"]:
raise ValueError("Runtime ZIP digest mismatch")
temporary.replace(archive)
unpacked = cache / "unpacked"
if not (cache / "complete").exists():
if unpacked.exists():
shutil.rmtree(unpacked)
safe_extract(archive, unpacked)
(cache / "complete").touch()
candidates = list(unpacked.glob("*/MANIFEST.json"))
if len(candidates) != 1:
raise ValueError("Invalid Eddy runtime ZIP")
source = candidates[0].parent
version = copy_runtime(source, root / "runtime")
dump(root / "toolchain.json", {"eddyVersion": version})
return version
def clean(source, commit):
if run(["git", "rev-parse", "HEAD"], cwd=source, capture=True) != commit:
raise ValueError("Product checkout changed commit during export")
run(["git", "-c", "core.fsmonitor=false", "diff", "--exit-code", "HEAD", "--"], cwd=source)
# New source files would not be part of the commit/source archive.
untracked = run(["git", "ls-files", "--others", "--exclude-standard"], cwd=source, capture=True)
if untracked:
raise ValueError("Build left untracked source files; use ignored/generated build paths")
def checkout(root, job):
if job.get("platform", "ios") != "ios" or job["recipe"].get("repoKey") != "tiktok-ios":
raise ValueError("Eddy relay currently supports iOS TikTok-family jobs only")
runtime(root)
source = root / "source"
# A clone owns its refs/worktree. Fetching cannot switch the implementation checkout.
run(["git", "clone", "--shared", "--no-checkout", job["repo"], source])
remote = run(["git", "remote", "get-url", "origin"], cwd=job["repo"], capture=True)
run(["git", "remote", "set-url", "origin", remote], cwd=source)
try:
run(["git", "cat-file", "-e", job["commit"] + "^{commit}"], cwd=source)
except subprocess.CalledProcessError:
run(["git", "fetch", "--no-tags", "origin", job["commit"]], cwd=source)
# The pinned commit on the branch's own name. The App is built in the node's shared checkout
# (eddy_export.ts pins it); this clone is what the source archive and relay files are cut from.
branch = job["start"]["branch"]
run(["git", "check-ref-format", "--branch", branch], cwd=source)
run(["git", "checkout", "-B", branch, job["commit"]], cwd=source)
(root / "build").mkdir()
clean(source, job["commit"])
def collect(root, job):
"""Accept what the build agent handed back: exactly the declared .app, and a receipt that names
the pinned commit. The receipt is the agent's declaration, recorded as BUILD.json; the source
archive is cut from the pinned commit regardless."""
clean(root / "source", job["commit"])
# The build ran in the node's shared checkout: it must still be the pinned commit, unmodified,
# or the App would not match the source archive.
clean(Path(job["repo"]), job["commit"])
recipe = job["recipe"]
receipt_path = root / "airbuild-receipt.json"
if not receipt_path.is_file():
raise ValueError("Build agent wrote no receipt")
receipt = json.loads(receipt_path.read_text())
if receipt.get("error"):
raise ValueError("Build agent reported a failure: " + str(receipt["error"])[:300])
if receipt.get("product") != recipe["product"] or receipt.get("commit") != job["commit"]:
raise ValueError("Build receipt does not name this export's product and pinned commit")
apps = sorted(p.name for p in (root / "build").iterdir() if p.suffix == ".app")
if apps != [recipe["app"]]:
raise ValueError("Expected exactly " + recipe["app"] + " in the build directory, found " + repr(apps))
app = root / "build" / recipe["app"]
if app.is_symlink() or not app.is_dir() or not (app / "Info.plist").is_file():
raise ValueError("Build did not produce a flat " + recipe["app"])
executable = plistlib.loads((app / "Info.plist").read_bytes()).get("CFBundleExecutable", "")
if not executable or Path(executable).name != executable or not (app / executable).is_file():
raise ValueError("Built App has no valid CFBundleExecutable")
dump(root / "built.json", {"app": str(app), "commit": job["commit"]})
def isolated(root, state):
"""Eddy and cat-supervisor default to machine-wide locations (/Users/Shared/DeviceSim,
~/iosapp) that this node's own agents use. An export must never switch or touch those.
DeviceSim's own release channel stays reachable: Eddy runtimes after 2026.10.23-eddy-home.18 carry no
DeviceSim and install the latest official stable into the job's DEVICESIM_ROOT.
EDDY_CLIENT marks its product usage `satellite`, which an Eddy runtime never uploads."""
return {"EDDY_STATE_DIR": str(root / state), "DEVICESIM_ROOT": str(root / "devicesim"),
"EDDY_STREAMLINE_URL": "http://127.0.0.1:1", "EDDY_STREAMLINE_TOKEN": "", "EDDY_CLIENT": "streamline-worker"}
# Eddy's build_app_baseline writes its work, stage and artifacts under one directory.
PACKAGE = "package"
def package(root, job):
home = root / "runtime"
# Run packaging imports with Eddy's bundled dependencies, not host site-packages.
python = home / "vendor/python/bin/python3"
if Path(sys.executable).resolve() != python.resolve():
env = dict(os.environ, PYTHONPATH=str(home / "vendor/site-packages"), **isolated(root, "package-state"))
run([python, __file__, "packaging", root], cwd=root, env=env)
return
sys.path[:0] = [str(home), str(home / "runtime/supervisor")]
from portable import build_app_baseline
from eddyctl import devicesim_install
import argparse, inspect
version = json.loads((root / "toolchain.json").read_text())["eddyVersion"]
if "extend" not in inspect.signature(build_app_baseline.build).parameters:
raise ValueError("Eddy runtime " + version + " cannot add export files to a baseline; needs 2026.10.15-eddy-home.12 or newer")
source = root / "source"
clean(source, job["commit"])
recipe = job["recipe"]
app = Path(json.loads((root / "built.json").read_text())["app"])
def extend(stage, record):
"""What an export adds to Eddy's baseline: task provenance, the build receipt, relay and install notes."""
record["origin"] = "streamline-task"
record["streamline"] = {"featureId": job["featureId"], "mrUrl": job["mrUrl"], "commit": job["commit"], "exportId": job["id"],
"branch": job.get("branch"), "start": job.get("start"), "relay": "RELAY.md"}
record["devicesimSha256"] = record["preparation"]["files"]["devicesim.dylib"]
record["build"] = {"builder": "airbuild-skill", "receipt": "BUILD.json"}
shutil.copy2(root / "airbuild-receipt.json", stage / "BUILD.json")
relay = job.get("instructions") or ("# " + job["title"] + "\n")
(stage / "RELAY.md").write_text(relay)
(root / "RELAY.md").write_text(relay)
(stage / "INSTALL.md").write_text(
"# " + job["title"] + "\n\nCommit: " + job["commit"] + "\nMR: " + job["mrUrl"] +
"\n\nDownload the App ZIP and the matching source ZIP into the same directory. "
"In Eddy > Baselines, enter the App ZIP path and install. Then create a task using this baseline. "
"Give RELAY.md to your local agent to continue with the requirements, progress and feedback history. "
"Sign in with your own account. Feature launch defaults are stored in baseline.json.\n"
"BUILD.json is the build agent's declaration of the remote Airbuild it ran. "
"The source ZIP is the original committed source.\n")
# Eddy's own baseline producer: cat-supervisor from the job's DeviceSim converts, injects and signs, so
# activate a DeviceSim under the job-local DEVICESIM_ROOT first: the one a runtime up to .18 ships in
# vendor/devicesim, else the latest official stable (newer runtimes carry none; Eddy then uses the active one).
devicesim_install.ensure_available()
vendored = home / "vendor/devicesim"
receipt = build_app_baseline.build(argparse.Namespace(
app=app, out=root / PACKAGE, source_repo=source, source_commit=job["commit"], release=version,
key=recipe["key"], tools=vendored if vendored.is_dir() else None, launch_defaults=recipe.get("launchDefaults")), extend=extend)
clean(source, job["commit"])
dump(root / "result.json", {"baselineId": receipt["baseline"], "commit": job["commit"], "eddyVersion": version,
"devicesimSha256": receipt["devicesimSha256"], "artifacts": {"app": receipt["app"], "source": receipt["source"]}})
def verify(root, job):
result = json.loads((root / "result.json").read_text())
artifacts = root / PACKAGE / "artifacts"
for artifact in result["artifacts"].values():
p = artifacts / artifact["file"]
if p.stat().st_size != artifact["bytes"] or digest(p) != artifact["sha256"]:
raise ValueError("Package digest mismatch")
with zipfile.ZipFile(artifacts / result["artifacts"]["source"]["file"]) as z:
source = json.loads(z.read(result["baselineId"] + "-source/SOURCE.json"))
if source["commit"] != job["commit"]:
raise ValueError("Source archive commit mismatch")
home = root / "runtime"
env = dict(os.environ, **isolated(root, "verify-state"))
def eddy(args):
return json.loads(run([home / "eddy", *args], cwd=home, env=env, capture=True, timeout=600))
started = False
try:
eddy(["setup"])
eddy(["baseline", "add", str(artifacts / result["artifacts"]["app"]["file"]), "--no-source", "--wait"])
eddy(["select", job["recipe"]["key"] + "@" + result["baselineId"]])
# The DeviceSim packaging activated under the job-local DEVICESIM_ROOT.
eddy(["devicesim", "select", "--baseline", result["baselineId"], "--version", "installed"])
started = True
eddy(["up", job["recipe"]["key"], "--wait"])
smoke = eddy(["smoke", job["recipe"]["key"]])
state = smoke.get("status", {})
if (not smoke.get("passed") or state.get("state") != "active" or state.get("editCount") != 0
or state.get("engineAttached") is not False or state.get("captureMode") != "on-demand"
or state.get("recording") is not False):
raise ValueError("Native smoke did not prove an active clean baseline")
image = Path(smoke["screenshot"])
if not image.is_file() or image.read_bytes()[:8] != b"\x89PNG\r\n\x1a\n":
raise ValueError("Native screenshot is missing")
shutil.copy2(image, root / "smoke.png")
dump(root / "verification.json", smoke)
finally:
if started:
eddy(["down", job["recipe"]["key"]])
def cleanup(root, job):
if not (root / "verify-state/installation.json").is_file():
return
env = dict(os.environ, **isolated(root, "verify-state"))
run([root / "runtime/eddy", "down", job["recipe"]["key"]], cwd=root / "runtime", env=env, timeout=45)
# The smoke screenshot spawned catuse's machine-wide capture daemon from the job's DeviceSim;
# Eddy stops that one (the next catuse call respawns it from the node's own DeviceSim).
run([root / "runtime/vendor/python/bin/python3", "-c", "from eddyctl import devicesim_install as d; d.stop_capture_daemon()"],
cwd=root / "runtime", env=env, timeout=30)
if __name__ == "__main__":
def stop(_signum, _frame):
raise InterruptedError("Export interrupted")
signal.signal(signal.SIGTERM, stop)
stage, directory = sys.argv[1:]
root = Path(directory).resolve()
job = json.loads((root / "job.json").read_text())
{"checkout": checkout, "collect": collect, "packaging": package, "verifying": verify, "cleanup": cleanup}[stage](root, job)