build(worker): package M4.8R3 shadow release

scripts/build_m48r3_worker_shadow_artifact.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+"""Build the deterministic M4.8R3 occupied-only Worker 006 shadow release."""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import hashlib
+import io
+import json
+import os
+import re
+import subprocess
+import tarfile
+import tempfile
+from pathlib import Path
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl"
+POWERSHELL_RUNNER = (
+    REPOSITORY_ROOT
+    / "experiments/perception/worker/Invoke-M48NNativeReferenceGraph.ps1"
+)
+PYTHON_RUNNER = (
+    REPOSITORY_ROOT
+    / "experiments/perception/run_m48s_reference_graph_shadow_worker.py"
+)
+NATIVE_CONFIG = (
+    REPOSITORY_ROOT / "experiments/perception/worker/rf_detr_large_native_kb4_config.pbtxt"
+)
+CONFIG_PATHS = (
+    Path("config/perception/m48r3-native-low-step-reference-graph-shadow-v1.json"),
+    Path("config/perception/m48r3-additive-low-step-occupancy-v1.json"),
+    Path("config/perception/m4-recorded-realtime-baseline-v1.json"),
+    Path("config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json"),
+    Path("config/perception/m4-geometry-association-v1.json"),
+    Path("config/perception/m4-temporal-motion-v1.json"),
+    Path("config/perception/m4-rolling-local-map-v1.json"),
+    Path("config/perception/m4-replay-threat-v3.json"),
+)
+PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$")
+
+
+class ArtifactBuildError(RuntimeError):
+    """The M4.8R3 release cannot be built from the declared source."""
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def git_revision(*, require_clean: bool) -> str:
+    result = subprocess.run(
+        ["git", "rev-parse", "HEAD"],
+        cwd=REPOSITORY_ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    revision = result.stdout.strip()
+    if re.fullmatch(r"[a-f0-9]{40}", revision) is None:
+        raise ArtifactBuildError("Git revision is not a full SHA-1")
+    if require_clean:
+        status = subprocess.run(
+            ["git", "status", "--porcelain"],
+            cwd=REPOSITORY_ROOT,
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        if status.stdout.strip():
+            raise ArtifactBuildError("Worker shadow artifact requires a clean worktree")
+    return revision
+
+
+def build_wheel(output: Path) -> Path:
+    environment = os.environ.copy()
+    environment["SOURCE_DATE_EPOCH"] = "0"
+    result = subprocess.run(
+        ["uv", "build", "--wheel", "--out-dir", str(output)],
+        cwd=REPOSITORY_ROOT,
+        env=environment,
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        detail = (result.stderr or result.stdout).strip()
+        raise ArtifactBuildError(f"wheel build failed: {detail}")
+    wheel = output / WHEEL_NAME
+    if not wheel.is_file() or wheel.is_symlink():
+        raise ArtifactBuildError("expected Worker wheel was not built")
+    return wheel
+
+
+def _tar_info(path: Path, arcname: str) -> tarfile.TarInfo:
+    info = tarfile.TarInfo(arcname)
+    info.uid = 0
+    info.gid = 0
+    info.uname = "root"
+    info.gname = "root"
+    info.mtime = 0
+    if path.is_dir():
+        info.type = tarfile.DIRTYPE
+        info.mode = 0o755
+    else:
+        info.type = tarfile.REGTYPE
+        info.mode = 0o644
+        info.size = path.stat().st_size
+    return info
+
+
+def write_canonical_archive(stage: Path, target: Path) -> None:
+    members = [stage / "manifest.env", stage / "files.txt", stage / "payload"]
+    members.extend(sorted((stage / "payload").rglob("*")))
+    target.parent.mkdir(parents=True, exist_ok=True)
+    with (
+        target.open("wb") as raw,
+        gzip.GzipFile(filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=0) as gz,
+        tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as archive,
+    ):
+        for path in members:
+            info = _tar_info(path, path.relative_to(stage).as_posix())
+            if path.is_file():
+                with path.open("rb") as stream:
+                    archive.addfile(info, stream)
+            else:
+                archive.addfile(info, io.BytesIO())
+
+
+def build_artifact(
+    patch_id: str,
+    output_directory: Path,
+    *,
+    revision: str | None = None,
+) -> dict[str, object]:
+    if PATCH_ID.fullmatch(patch_id) is None:
+        raise ArtifactBuildError("patch id is invalid")
+    selected_revision = revision or git_revision(require_clean=True)
+    if re.fullmatch(r"[a-f0-9]{40}", selected_revision) is None:
+        raise ArtifactBuildError("artifact revision is invalid")
+    sources = (
+        POWERSHELL_RUNNER,
+        PYTHON_RUNNER,
+        NATIVE_CONFIG,
+        *(REPOSITORY_ROOT / relative for relative in CONFIG_PATHS),
+    )
+    if any(path.is_symlink() or not path.is_file() for path in sources):
+        raise ArtifactBuildError("Worker release input is not a regular file")
+    with tempfile.TemporaryDirectory(prefix="mission-core-m48r3-worker-") as directory:
+        stage = Path(directory)
+        payload = stage / "payload"
+        payload.mkdir()
+        wheel = build_wheel(stage / "wheel")
+        copied: list[Path] = []
+        for source in sources:
+            destination = payload / source.name
+            destination.write_bytes(source.read_bytes())
+            copied.append(destination)
+        wheel_destination = payload / WHEEL_NAME
+        wheel_destination.write_bytes(wheel.read_bytes())
+        copied.append(wheel_destination)
+        release = {
+            "schema_version": "missioncore.m48r3-worker-shadow-release/v1",
+            "patch_id": patch_id,
+            "transition": "m48r3-additive-low-step-isolated-shadow/v1",
+            "code_revision": selected_revision,
+            "worker_id": "worker-006",
+            "graph_id": "reference-perception-graph/v2",
+            "detector_provider_id": (
+                "triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0"
+            ),
+            "geometry_provider_id": "ravnoves00-additive-low-step-geometry/v1",
+            "expected_frames": 4489,
+            "requested_source_rate_hz": 12.0,
+            "candidate_accepted": False,
+            "production_accepted": False,
+            "durable_worker_action": "none",
+            "canonical_triton_action": "none",
+            "files": {
+                path.name: {"sha256": sha256_file(path), "bytes": path.stat().st_size}
+                for path in sorted(copied)
+            },
+        }
+        release_path = payload / "release.json"
+        release_path.write_text(
+            json.dumps(release, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+            "utf-8",
+        )
+        payload_files = sorted((*release["files"], release_path.name))
+        (stage / "manifest.env").write_text(
+            f"id={patch_id}\ncomponent=mission-core-worker\ntype=shadow-release\n",
+            "utf-8",
+        )
+        (stage / "files.txt").write_text("\n".join(payload_files) + "\n", "utf-8")
+        target = output_directory.resolve() / f"nodedc-{patch_id}.tgz"
+        write_canonical_archive(stage, target)
+    return {
+        "ok": True,
+        "patch_id": patch_id,
+        "artifact": str(target),
+        "sha256": sha256_file(target),
+        "code_revision": selected_revision,
+        "wheel_sha256": release["files"][WHEEL_NAME]["sha256"],
+        "payload_files": payload_files,
+        "transition": release["transition"],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("patch_id")
+    parser.add_argument(
+        "--output-directory",
+        type=Path,
+        default=REPOSITORY_ROOT / ".runtime/worker-artifacts",
+    )
+    arguments = parser.parse_args()
+    try:
+        result = build_artifact(arguments.patch_id, arguments.output_directory)
+    except (ArtifactBuildError, OSError, subprocess.SubprocessError) as exc:
+        parser.error(str(exc))
+    print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_m48r3_worker_shadow_artifact.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+import tarfile
+from pathlib import Path
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+BUILDER_PATH = REPOSITORY_ROOT / "scripts/build_m48r3_worker_shadow_artifact.py"
+SPEC = importlib.util.spec_from_file_location("m48r3_worker_shadow_builder", BUILDER_PATH)
+assert SPEC is not None and SPEC.loader is not None
+BUILDER = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(BUILDER)
+
+
+def _sha256(value: bytes) -> str:
+    return hashlib.sha256(value).hexdigest()
+
+
+def _regular_files(archive: tarfile.TarFile) -> dict[str, bytes]:
+    result: dict[str, bytes] = {}
+    for member in archive.getmembers():
+        if not member.isfile():
+            continue
+        stream = archive.extractfile(member)
+        assert stream is not None
+        result[member.name] = stream.read()
+    return result
+
+
+def test_m48r3_worker_artifact_is_deterministic_and_bounded(tmp_path: Path) -> None:
+    patch_id = "mission-core-m48r3-low-step-unit-001"
+    revision = "a" * 40
+    first = BUILDER.build_artifact(patch_id, tmp_path / "first", revision=revision)
+    second = BUILDER.build_artifact(patch_id, tmp_path / "second", revision=revision)
+
+    first_bytes = Path(first["artifact"]).read_bytes()
+    assert first_bytes == Path(second["artifact"]).read_bytes()
+    assert first["sha256"] == _sha256(first_bytes)
+    with tarfile.open(first["artifact"], "r:gz") as archive:
+        members = archive.getmembers()
+        regular = _regular_files(archive)
+    assert all(not member.issym() and not member.islnk() for member in members)
+    assert set(regular) == {
+        "manifest.env",
+        "files.txt",
+        *(f"payload/{name}" for name in first["payload_files"]),
+    }
+    assert regular["files.txt"].decode().splitlines() == first["payload_files"]
+    release = json.loads(regular["payload/release.json"])
+    assert release["code_revision"] == revision
+    assert release["geometry_provider_id"] == (
+        "ravnoves00-additive-low-step-geometry/v1"
+    )
+    assert release["candidate_accepted"] is False
+    assert release["production_accepted"] is False
+    assert release["durable_worker_action"] == "none"
+    assert release["canonical_triton_action"] == "none"
+    serialized = json.dumps(release).lower()
+    assert "private key" not in serialized
+    assert "password=" not in serialized