Initial import of streamline
This commit is contained in:
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Per-export checkout/collect/package/smoke worker. The build between checkout and collect is an
|
||||
agent following DeviceSim's airbuild skill (satellite/src/eddy_export_build.ts); this file only
|
||||
checks what it hands back. No global Eddy installation, DeviceSim selection or product checkout
|
||||
mutations."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import plistlib
|
||||
import shutil
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
import zipfile
|
||||
|
||||
OFFICIAL = "https://voffline.byted.org/download/tos/schedule/tiktok-social-bot-config/eddy/"
|
||||
|
||||
def digest(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
def dump(path, value):
|
||||
Path(path).write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
|
||||
|
||||
def run(args, cwd=None, env=None, capture=False, timeout=7200):
|
||||
result = subprocess.run(list(map(str, args)), cwd=cwd, env=env, check=True, timeout=timeout,
|
||||
stdout=subprocess.PIPE if capture else None, text=capture)
|
||||
return result.stdout.strip() if capture else None
|
||||
|
||||
def inside(root, path):
|
||||
root, path = Path(root).resolve(), Path(path).resolve()
|
||||
if path == root or root not in path.parents:
|
||||
raise ValueError("Path escapes owned export directory: " + str(path))
|
||||
return path
|
||||
|
||||
def read_url(url):
|
||||
if not url.startswith(OFFICIAL) or any(c in url for c in ["\n", "\r", "?", "#"]):
|
||||
raise ValueError("Expected an official Eddy distribution URL")
|
||||
with urllib.request.urlopen(url, timeout=90) as response:
|
||||
if not response.geturl().startswith(OFFICIAL):
|
||||
raise ValueError("Unexpected distribution redirect")
|
||||
return response.read(1024 * 1024)
|
||||
|
||||
def safe_extract(archive, destination):
|
||||
"""Validate paths and symlink targets before preserving the runtime's executable bits."""
|
||||
root = Path(destination)
|
||||
with zipfile.ZipFile(archive) as z:
|
||||
entries = [i for i in z.infolist() if not i.filename.startswith("__MACOSX/")]
|
||||
links = {i.filename.rstrip("/") for i in entries if stat.S_ISLNK(i.external_attr >> 16)}
|
||||
for i in entries:
|
||||
rel = Path(i.filename)
|
||||
inside(root, root / rel)
|
||||
if rel.is_absolute() or ".." in rel.parts or any(str(p) in links for p in rel.parents):
|
||||
raise ValueError("Unsafe ZIP entry")
|
||||
for i in entries:
|
||||
dest = root / i.filename
|
||||
if i.is_dir():
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
continue
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
mode = i.external_attr >> 16
|
||||
if stat.S_ISLNK(mode):
|
||||
target = z.read(i).decode()
|
||||
inside(root, dest.parent / target)
|
||||
dest.symlink_to(target)
|
||||
else:
|
||||
with z.open(i) as source, dest.open("wb") as output:
|
||||
shutil.copyfileobj(source, output)
|
||||
dest.chmod(stat.S_IMODE(mode) or 0o644)
|
||||
|
||||
def copy_runtime(source, dest):
|
||||
"""Copy manifest-listed distribution files only, excluding installed apps, accounts and tasks."""
|
||||
manifest = json.loads((source / "MANIFEST.json").read_text())
|
||||
if manifest.get("kind") != "runtime":
|
||||
raise ValueError("Expected standalone Eddy runtime distribution")
|
||||
dest.mkdir(parents=True)
|
||||
for entry in manifest["files"]:
|
||||
p = source / entry["path"]
|
||||
q = inside(dest, dest / entry["path"])
|
||||
inside(source, p)
|
||||
q.parent.mkdir(parents=True, exist_ok=True)
|
||||
if "symlink" in entry:
|
||||
if not p.is_symlink() or os.readlink(p) != entry["symlink"]:
|
||||
raise ValueError("Runtime symlink mismatch")
|
||||
inside(dest, q.parent / entry["symlink"])
|
||||
q.symlink_to(entry["symlink"])
|
||||
elif "sha256" in entry:
|
||||
if digest(p) != entry["sha256"]:
|
||||
raise ValueError("Runtime file digest mismatch: " + entry["path"])
|
||||
shutil.copy2(p, q)
|
||||
shutil.copy2(source / "MANIFEST.json", dest / "MANIFEST.json")
|
||||
return manifest["release"]
|
||||
|
||||
def runtime(root):
|
||||
url = read_url(OFFICIAL + "LATEST").decode().strip()
|
||||
metadata = json.loads(read_url(url.rsplit("/", 1)[0] + "/release.json"))
|
||||
artifact = metadata["artifact"]
|
||||
if (artifact.get("url") != url or not url.startswith(OFFICIAL) or artifact.get("kind") != "runtime"
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", artifact.get("sha256", ""))
|
||||
or type(artifact.get("bytes")) is not int or not 0 < artifact["bytes"] <= 1024 ** 3):
|
||||
raise ValueError("Eddy release pointer mismatch")
|
||||
cache = Path.home() / ".cache/streamline/eddy" / artifact["sha256"]
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
archive = cache / "runtime.zip"
|
||||
if not archive.is_file() or archive.stat().st_size != artifact["bytes"] or digest(archive) != artifact["sha256"]:
|
||||
temporary = cache / "runtime.pending"
|
||||
with urllib.request.urlopen(url, timeout=90) as response, temporary.open("wb") as output:
|
||||
if not response.geturl().startswith(OFFICIAL):
|
||||
raise ValueError("Unexpected runtime redirect")
|
||||
shutil.copyfileobj(response, output, 1024 * 1024)
|
||||
if temporary.stat().st_size != artifact["bytes"] or digest(temporary) != artifact["sha256"]:
|
||||
raise ValueError("Runtime ZIP digest mismatch")
|
||||
temporary.replace(archive)
|
||||
unpacked = cache / "unpacked"
|
||||
if not (cache / "complete").exists():
|
||||
if unpacked.exists():
|
||||
shutil.rmtree(unpacked)
|
||||
safe_extract(archive, unpacked)
|
||||
(cache / "complete").touch()
|
||||
candidates = list(unpacked.glob("*/MANIFEST.json"))
|
||||
if len(candidates) != 1:
|
||||
raise ValueError("Invalid Eddy runtime ZIP")
|
||||
source = candidates[0].parent
|
||||
version = copy_runtime(source, root / "runtime")
|
||||
dump(root / "toolchain.json", {"eddyVersion": version})
|
||||
return version
|
||||
|
||||
def clean(source, commit):
|
||||
if run(["git", "rev-parse", "HEAD"], cwd=source, capture=True) != commit:
|
||||
raise ValueError("Product checkout changed commit during export")
|
||||
run(["git", "-c", "core.fsmonitor=false", "diff", "--exit-code", "HEAD", "--"], cwd=source)
|
||||
# New source files would not be part of the commit/source archive.
|
||||
untracked = run(["git", "ls-files", "--others", "--exclude-standard"], cwd=source, capture=True)
|
||||
if untracked:
|
||||
raise ValueError("Build left untracked source files; use ignored/generated build paths")
|
||||
|
||||
def checkout(root, job):
|
||||
if job.get("platform", "ios") != "ios" or job["recipe"].get("repoKey") != "tiktok-ios":
|
||||
raise ValueError("Eddy relay currently supports iOS TikTok-family jobs only")
|
||||
runtime(root)
|
||||
source = root / "source"
|
||||
# A clone owns its refs/worktree. Fetching cannot switch the implementation checkout.
|
||||
run(["git", "clone", "--shared", "--no-checkout", job["repo"], source])
|
||||
remote = run(["git", "remote", "get-url", "origin"], cwd=job["repo"], capture=True)
|
||||
run(["git", "remote", "set-url", "origin", remote], cwd=source)
|
||||
try:
|
||||
run(["git", "cat-file", "-e", job["commit"] + "^{commit}"], cwd=source)
|
||||
except subprocess.CalledProcessError:
|
||||
run(["git", "fetch", "--no-tags", "origin", job["commit"]], cwd=source)
|
||||
# The pinned commit on the branch's own name. The App is built in the node's shared checkout
|
||||
# (eddy_export.ts pins it); this clone is what the source archive and relay files are cut from.
|
||||
branch = job["start"]["branch"]
|
||||
run(["git", "check-ref-format", "--branch", branch], cwd=source)
|
||||
run(["git", "checkout", "-B", branch, job["commit"]], cwd=source)
|
||||
(root / "build").mkdir()
|
||||
clean(source, job["commit"])
|
||||
|
||||
def collect(root, job):
|
||||
"""Accept what the build agent handed back: exactly the declared .app, and a receipt that names
|
||||
the pinned commit. The receipt is the agent's declaration, recorded as BUILD.json; the source
|
||||
archive is cut from the pinned commit regardless."""
|
||||
clean(root / "source", job["commit"])
|
||||
# The build ran in the node's shared checkout: it must still be the pinned commit, unmodified,
|
||||
# or the App would not match the source archive.
|
||||
clean(Path(job["repo"]), job["commit"])
|
||||
recipe = job["recipe"]
|
||||
receipt_path = root / "airbuild-receipt.json"
|
||||
if not receipt_path.is_file():
|
||||
raise ValueError("Build agent wrote no receipt")
|
||||
receipt = json.loads(receipt_path.read_text())
|
||||
if receipt.get("error"):
|
||||
raise ValueError("Build agent reported a failure: " + str(receipt["error"])[:300])
|
||||
if receipt.get("product") != recipe["product"] or receipt.get("commit") != job["commit"]:
|
||||
raise ValueError("Build receipt does not name this export's product and pinned commit")
|
||||
apps = sorted(p.name for p in (root / "build").iterdir() if p.suffix == ".app")
|
||||
if apps != [recipe["app"]]:
|
||||
raise ValueError("Expected exactly " + recipe["app"] + " in the build directory, found " + repr(apps))
|
||||
app = root / "build" / recipe["app"]
|
||||
if app.is_symlink() or not app.is_dir() or not (app / "Info.plist").is_file():
|
||||
raise ValueError("Build did not produce a flat " + recipe["app"])
|
||||
executable = plistlib.loads((app / "Info.plist").read_bytes()).get("CFBundleExecutable", "")
|
||||
if not executable or Path(executable).name != executable or not (app / executable).is_file():
|
||||
raise ValueError("Built App has no valid CFBundleExecutable")
|
||||
dump(root / "built.json", {"app": str(app), "commit": job["commit"]})
|
||||
|
||||
def isolated(root, state):
|
||||
"""Eddy and cat-supervisor default to machine-wide locations (/Users/Shared/DeviceSim,
|
||||
~/iosapp) that this node's own agents use. An export must never switch or touch those.
|
||||
DeviceSim's own release channel stays reachable: Eddy runtimes after 2026.10.23-eddy-home.18 carry no
|
||||
DeviceSim and install the latest official stable into the job's DEVICESIM_ROOT.
|
||||
EDDY_CLIENT marks its product usage `satellite`, which an Eddy runtime never uploads."""
|
||||
return {"EDDY_STATE_DIR": str(root / state), "DEVICESIM_ROOT": str(root / "devicesim"),
|
||||
"EDDY_STREAMLINE_URL": "http://127.0.0.1:1", "EDDY_STREAMLINE_TOKEN": "", "EDDY_CLIENT": "streamline-worker"}
|
||||
|
||||
# Eddy's build_app_baseline writes its work, stage and artifacts under one directory.
|
||||
PACKAGE = "package"
|
||||
|
||||
def package(root, job):
|
||||
home = root / "runtime"
|
||||
# Run packaging imports with Eddy's bundled dependencies, not host site-packages.
|
||||
python = home / "vendor/python/bin/python3"
|
||||
if Path(sys.executable).resolve() != python.resolve():
|
||||
env = dict(os.environ, PYTHONPATH=str(home / "vendor/site-packages"), **isolated(root, "package-state"))
|
||||
run([python, __file__, "packaging", root], cwd=root, env=env)
|
||||
return
|
||||
sys.path[:0] = [str(home), str(home / "runtime/supervisor")]
|
||||
from portable import build_app_baseline
|
||||
from eddyctl import devicesim_install
|
||||
import argparse, inspect
|
||||
version = json.loads((root / "toolchain.json").read_text())["eddyVersion"]
|
||||
if "extend" not in inspect.signature(build_app_baseline.build).parameters:
|
||||
raise ValueError("Eddy runtime " + version + " cannot add export files to a baseline; needs 2026.10.15-eddy-home.12 or newer")
|
||||
source = root / "source"
|
||||
clean(source, job["commit"])
|
||||
recipe = job["recipe"]
|
||||
app = Path(json.loads((root / "built.json").read_text())["app"])
|
||||
|
||||
def extend(stage, record):
|
||||
"""What an export adds to Eddy's baseline: task provenance, the build receipt, relay and install notes."""
|
||||
record["origin"] = "streamline-task"
|
||||
record["streamline"] = {"featureId": job["featureId"], "mrUrl": job["mrUrl"], "commit": job["commit"], "exportId": job["id"],
|
||||
"branch": job.get("branch"), "start": job.get("start"), "relay": "RELAY.md"}
|
||||
record["devicesimSha256"] = record["preparation"]["files"]["devicesim.dylib"]
|
||||
record["build"] = {"builder": "airbuild-skill", "receipt": "BUILD.json"}
|
||||
shutil.copy2(root / "airbuild-receipt.json", stage / "BUILD.json")
|
||||
relay = job.get("instructions") or ("# " + job["title"] + "\n")
|
||||
(stage / "RELAY.md").write_text(relay)
|
||||
(root / "RELAY.md").write_text(relay)
|
||||
(stage / "INSTALL.md").write_text(
|
||||
"# " + job["title"] + "\n\nCommit: " + job["commit"] + "\nMR: " + job["mrUrl"] +
|
||||
"\n\nDownload the App ZIP and the matching source ZIP into the same directory. "
|
||||
"In Eddy > Baselines, enter the App ZIP path and install. Then create a task using this baseline. "
|
||||
"Give RELAY.md to your local agent to continue with the requirements, progress and feedback history. "
|
||||
"Sign in with your own account. Feature launch defaults are stored in baseline.json.\n"
|
||||
"BUILD.json is the build agent's declaration of the remote Airbuild it ran. "
|
||||
"The source ZIP is the original committed source.\n")
|
||||
|
||||
# Eddy's own baseline producer: cat-supervisor from the job's DeviceSim converts, injects and signs, so
|
||||
# activate a DeviceSim under the job-local DEVICESIM_ROOT first: the one a runtime up to .18 ships in
|
||||
# vendor/devicesim, else the latest official stable (newer runtimes carry none; Eddy then uses the active one).
|
||||
devicesim_install.ensure_available()
|
||||
vendored = home / "vendor/devicesim"
|
||||
receipt = build_app_baseline.build(argparse.Namespace(
|
||||
app=app, out=root / PACKAGE, source_repo=source, source_commit=job["commit"], release=version,
|
||||
key=recipe["key"], tools=vendored if vendored.is_dir() else None, launch_defaults=recipe.get("launchDefaults")), extend=extend)
|
||||
clean(source, job["commit"])
|
||||
dump(root / "result.json", {"baselineId": receipt["baseline"], "commit": job["commit"], "eddyVersion": version,
|
||||
"devicesimSha256": receipt["devicesimSha256"], "artifacts": {"app": receipt["app"], "source": receipt["source"]}})
|
||||
|
||||
def verify(root, job):
|
||||
result = json.loads((root / "result.json").read_text())
|
||||
artifacts = root / PACKAGE / "artifacts"
|
||||
for artifact in result["artifacts"].values():
|
||||
p = artifacts / artifact["file"]
|
||||
if p.stat().st_size != artifact["bytes"] or digest(p) != artifact["sha256"]:
|
||||
raise ValueError("Package digest mismatch")
|
||||
with zipfile.ZipFile(artifacts / result["artifacts"]["source"]["file"]) as z:
|
||||
source = json.loads(z.read(result["baselineId"] + "-source/SOURCE.json"))
|
||||
if source["commit"] != job["commit"]:
|
||||
raise ValueError("Source archive commit mismatch")
|
||||
home = root / "runtime"
|
||||
env = dict(os.environ, **isolated(root, "verify-state"))
|
||||
def eddy(args):
|
||||
return json.loads(run([home / "eddy", *args], cwd=home, env=env, capture=True, timeout=600))
|
||||
started = False
|
||||
try:
|
||||
eddy(["setup"])
|
||||
eddy(["baseline", "add", str(artifacts / result["artifacts"]["app"]["file"]), "--no-source", "--wait"])
|
||||
eddy(["select", job["recipe"]["key"] + "@" + result["baselineId"]])
|
||||
# The DeviceSim packaging activated under the job-local DEVICESIM_ROOT.
|
||||
eddy(["devicesim", "select", "--baseline", result["baselineId"], "--version", "installed"])
|
||||
started = True
|
||||
eddy(["up", job["recipe"]["key"], "--wait"])
|
||||
smoke = eddy(["smoke", job["recipe"]["key"]])
|
||||
state = smoke.get("status", {})
|
||||
if (not smoke.get("passed") or state.get("state") != "active" or state.get("editCount") != 0
|
||||
or state.get("engineAttached") is not False or state.get("captureMode") != "on-demand"
|
||||
or state.get("recording") is not False):
|
||||
raise ValueError("Native smoke did not prove an active clean baseline")
|
||||
image = Path(smoke["screenshot"])
|
||||
if not image.is_file() or image.read_bytes()[:8] != b"\x89PNG\r\n\x1a\n":
|
||||
raise ValueError("Native screenshot is missing")
|
||||
shutil.copy2(image, root / "smoke.png")
|
||||
dump(root / "verification.json", smoke)
|
||||
finally:
|
||||
if started:
|
||||
eddy(["down", job["recipe"]["key"]])
|
||||
|
||||
def cleanup(root, job):
|
||||
if not (root / "verify-state/installation.json").is_file():
|
||||
return
|
||||
env = dict(os.environ, **isolated(root, "verify-state"))
|
||||
run([root / "runtime/eddy", "down", job["recipe"]["key"]], cwd=root / "runtime", env=env, timeout=45)
|
||||
# The smoke screenshot spawned catuse's machine-wide capture daemon from the job's DeviceSim;
|
||||
# Eddy stops that one (the next catuse call respawns it from the node's own DeviceSim).
|
||||
run([root / "runtime/vendor/python/bin/python3", "-c", "from eddyctl import devicesim_install as d; d.stop_capture_daemon()"],
|
||||
cwd=root / "runtime", env=env, timeout=30)
|
||||
|
||||
if __name__ == "__main__":
|
||||
def stop(_signum, _frame):
|
||||
raise InterruptedError("Export interrupted")
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
stage, directory = sys.argv[1:]
|
||||
root = Path(directory).resolve()
|
||||
job = json.loads((root / "job.json").read_text())
|
||||
{"checkout": checkout, "collect": collect, "packaging": package, "verifying": verify, "cleanup": cleanup}[stage](root, job)
|
||||
Reference in New Issue
Block a user