diff --git a/scripts/build_m48r3_worker_shadow_artifact.py b/scripts/build_m48r3_worker_shadow_artifact.py new file mode 100644 index 0000000..f393788 --- /dev/null +++ b/scripts/build_m48r3_worker_shadow_artifact.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Build the deterministic M4.8R3 occupied-only Worker 006 shadow release.""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import os +import re +import subprocess +import tarfile +import tempfile +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl" +POWERSHELL_RUNNER = ( + REPOSITORY_ROOT + / "experiments/perception/worker/Invoke-M48NNativeReferenceGraph.ps1" +) +PYTHON_RUNNER = ( + REPOSITORY_ROOT + / "experiments/perception/run_m48s_reference_graph_shadow_worker.py" +) +NATIVE_CONFIG = ( + REPOSITORY_ROOT / "experiments/perception/worker/rf_detr_large_native_kb4_config.pbtxt" +) +CONFIG_PATHS = ( + Path("config/perception/m48r3-native-low-step-reference-graph-shadow-v1.json"), + Path("config/perception/m48r3-additive-low-step-occupancy-v1.json"), + Path("config/perception/m4-recorded-realtime-baseline-v1.json"), + Path("config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json"), + Path("config/perception/m4-geometry-association-v1.json"), + Path("config/perception/m4-temporal-motion-v1.json"), + Path("config/perception/m4-rolling-local-map-v1.json"), + Path("config/perception/m4-replay-threat-v3.json"), +) +PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$") + + +class ArtifactBuildError(RuntimeError): + """The M4.8R3 release cannot be built from the declared source.""" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_revision(*, require_clean: bool) -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=REPOSITORY_ROOT, + check=True, + capture_output=True, + text=True, + ) + revision = result.stdout.strip() + if re.fullmatch(r"[a-f0-9]{40}", revision) is None: + raise ArtifactBuildError("Git revision is not a full SHA-1") + if require_clean: + status = subprocess.run( + ["git", "status", "--porcelain"], + cwd=REPOSITORY_ROOT, + check=True, + capture_output=True, + text=True, + ) + if status.stdout.strip(): + raise ArtifactBuildError("Worker shadow artifact requires a clean worktree") + return revision + + +def build_wheel(output: Path) -> Path: + environment = os.environ.copy() + environment["SOURCE_DATE_EPOCH"] = "0" + result = subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(output)], + cwd=REPOSITORY_ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + detail = (result.stderr or result.stdout).strip() + raise ArtifactBuildError(f"wheel build failed: {detail}") + wheel = output / WHEEL_NAME + if not wheel.is_file() or wheel.is_symlink(): + raise ArtifactBuildError("expected Worker wheel was not built") + return wheel + + +def _tar_info(path: Path, arcname: str) -> tarfile.TarInfo: + info = tarfile.TarInfo(arcname) + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + info.mtime = 0 + if path.is_dir(): + info.type = tarfile.DIRTYPE + info.mode = 0o755 + else: + info.type = tarfile.REGTYPE + info.mode = 0o644 + info.size = path.stat().st_size + return info + + +def write_canonical_archive(stage: Path, target: Path) -> None: + members = [stage / "manifest.env", stage / "files.txt", stage / "payload"] + members.extend(sorted((stage / "payload").rglob("*"))) + target.parent.mkdir(parents=True, exist_ok=True) + with ( + target.open("wb") as raw, + gzip.GzipFile(filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=0) as gz, + tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as archive, + ): + for path in members: + info = _tar_info(path, path.relative_to(stage).as_posix()) + if path.is_file(): + with path.open("rb") as stream: + archive.addfile(info, stream) + else: + archive.addfile(info, io.BytesIO()) + + +def build_artifact( + patch_id: str, + output_directory: Path, + *, + revision: str | None = None, +) -> dict[str, object]: + if PATCH_ID.fullmatch(patch_id) is None: + raise ArtifactBuildError("patch id is invalid") + selected_revision = revision or git_revision(require_clean=True) + if re.fullmatch(r"[a-f0-9]{40}", selected_revision) is None: + raise ArtifactBuildError("artifact revision is invalid") + sources = ( + POWERSHELL_RUNNER, + PYTHON_RUNNER, + NATIVE_CONFIG, + *(REPOSITORY_ROOT / relative for relative in CONFIG_PATHS), + ) + if any(path.is_symlink() or not path.is_file() for path in sources): + raise ArtifactBuildError("Worker release input is not a regular file") + with tempfile.TemporaryDirectory(prefix="mission-core-m48r3-worker-") as directory: + stage = Path(directory) + payload = stage / "payload" + payload.mkdir() + wheel = build_wheel(stage / "wheel") + copied: list[Path] = [] + for source in sources: + destination = payload / source.name + destination.write_bytes(source.read_bytes()) + copied.append(destination) + wheel_destination = payload / WHEEL_NAME + wheel_destination.write_bytes(wheel.read_bytes()) + copied.append(wheel_destination) + release = { + "schema_version": "missioncore.m48r3-worker-shadow-release/v1", + "patch_id": patch_id, + "transition": "m48r3-additive-low-step-isolated-shadow/v1", + "code_revision": selected_revision, + "worker_id": "worker-006", + "graph_id": "reference-perception-graph/v2", + "detector_provider_id": ( + "triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0" + ), + "geometry_provider_id": "ravnoves00-additive-low-step-geometry/v1", + "expected_frames": 4489, + "requested_source_rate_hz": 12.0, + "candidate_accepted": False, + "production_accepted": False, + "durable_worker_action": "none", + "canonical_triton_action": "none", + "files": { + path.name: {"sha256": sha256_file(path), "bytes": path.stat().st_size} + for path in sorted(copied) + }, + } + release_path = payload / "release.json" + release_path.write_text( + json.dumps(release, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + "utf-8", + ) + payload_files = sorted((*release["files"], release_path.name)) + (stage / "manifest.env").write_text( + f"id={patch_id}\ncomponent=mission-core-worker\ntype=shadow-release\n", + "utf-8", + ) + (stage / "files.txt").write_text("\n".join(payload_files) + "\n", "utf-8") + target = output_directory.resolve() / f"nodedc-{patch_id}.tgz" + write_canonical_archive(stage, target) + return { + "ok": True, + "patch_id": patch_id, + "artifact": str(target), + "sha256": sha256_file(target), + "code_revision": selected_revision, + "wheel_sha256": release["files"][WHEEL_NAME]["sha256"], + "payload_files": payload_files, + "transition": release["transition"], + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("patch_id") + parser.add_argument( + "--output-directory", + type=Path, + default=REPOSITORY_ROOT / ".runtime/worker-artifacts", + ) + arguments = parser.parse_args() + try: + result = build_artifact(arguments.patch_id, arguments.output_directory) + except (ArtifactBuildError, OSError, subprocess.SubprocessError) as exc: + parser.error(str(exc)) + print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_m48r3_worker_shadow_artifact.py b/tests/test_m48r3_worker_shadow_artifact.py new file mode 100644 index 0000000..ee7d342 --- /dev/null +++ b/tests/test_m48r3_worker_shadow_artifact.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import tarfile +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +BUILDER_PATH = REPOSITORY_ROOT / "scripts/build_m48r3_worker_shadow_artifact.py" +SPEC = importlib.util.spec_from_file_location("m48r3_worker_shadow_builder", BUILDER_PATH) +assert SPEC is not None and SPEC.loader is not None +BUILDER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(BUILDER) + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _regular_files(archive: tarfile.TarFile) -> dict[str, bytes]: + result: dict[str, bytes] = {} + for member in archive.getmembers(): + if not member.isfile(): + continue + stream = archive.extractfile(member) + assert stream is not None + result[member.name] = stream.read() + return result + + +def test_m48r3_worker_artifact_is_deterministic_and_bounded(tmp_path: Path) -> None: + patch_id = "mission-core-m48r3-low-step-unit-001" + revision = "a" * 40 + first = BUILDER.build_artifact(patch_id, tmp_path / "first", revision=revision) + second = BUILDER.build_artifact(patch_id, tmp_path / "second", revision=revision) + + first_bytes = Path(first["artifact"]).read_bytes() + assert first_bytes == Path(second["artifact"]).read_bytes() + assert first["sha256"] == _sha256(first_bytes) + with tarfile.open(first["artifact"], "r:gz") as archive: + members = archive.getmembers() + regular = _regular_files(archive) + assert all(not member.issym() and not member.islnk() for member in members) + assert set(regular) == { + "manifest.env", + "files.txt", + *(f"payload/{name}" for name in first["payload_files"]), + } + assert regular["files.txt"].decode().splitlines() == first["payload_files"] + release = json.loads(regular["payload/release.json"]) + assert release["code_revision"] == revision + assert release["geometry_provider_id"] == ( + "ravnoves00-additive-low-step-geometry/v1" + ) + assert release["candidate_accepted"] is False + assert release["production_accepted"] is False + assert release["durable_worker_action"] == "none" + assert release["canonical_triton_action"] == "none" + serialized = json.dumps(release).lower() + assert "private key" not in serialized + assert "password=" not in serialized