From 720487379cbbf2bb97a43e2c560fe571b4fcb518 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Wed, 5 Aug 2026 15:09:34 +0300 Subject: [PATCH] feat(deploy): build worker shadow artifact --- ...ission-core-worker-shadow-v1.template.json | 143 ++++++++++++++ scripts/build_m4_worker_shadow_artifact.py | 186 ++++++++++++++++++ tests/test_m4_worker_shadow_artifact.py | 129 ++++++++++++ 3 files changed, 458 insertions(+) create mode 100644 config/deployment/mission-core-worker-shadow-v1.template.json create mode 100644 scripts/build_m4_worker_shadow_artifact.py create mode 100644 tests/test_m4_worker_shadow_artifact.py diff --git a/config/deployment/mission-core-worker-shadow-v1.template.json b/config/deployment/mission-core-worker-shadow-v1.template.json new file mode 100644 index 0000000..904dd38 --- /dev/null +++ b/config/deployment/mission-core-worker-shadow-v1.template.json @@ -0,0 +1,143 @@ +{ + "acceptance": { + "class_routing_used": false, + "expected_frames": 4489, + "failed_frames": 0, + "minimum_end_to_end_fps": 10.004 + }, + "artifact_type": "shadow-release", + "code_revision": "__CODE_REVISION__", + "component": "mission-core-worker", + "container": { + "cap_drop": [ + "ALL" + ], + "image_id": "sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", + "image_ref": "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", + "name": "ndc-mission-core-m4-detector-shadow", + "network_container": "ndc-mission-core-triton", + "no_new_privileges": true, + "public_ports": false, + "python_path": "/release/nodedc_mission_core-0.1.0-py3-none-any.whl:/opt/media:/opt/opencv:/opt/pillow", + "read_only_rootfs": true, + "tmpfs": "/tmp:rw,noexec,nosuid,size=2g", + "triton_origin": "http://127.0.0.1:8000" + }, + "dependencies": [ + { + "bytes": 111694018, + "container_path": "/opt/media", + "file_count": 337, + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\derived\\perception-e15-media-pyav180-lz445-v1", + "id": "pyav-media", + "includes": [ + "." + ], + "tree_sha256": "301f5b9334703f838edc1b682c25ed81086bf3fe81a838c4c849e08e75bcf8e2" + }, + { + "bytes": 155757763, + "container_path": "/opt/opencv", + "file_count": 119, + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\derived\\perception-e3-opencv413092-v1\\packages", + "id": "opencv-packages", + "includes": [ + "." + ], + "tree_sha256": "abdc6cb3989e38b0c2f4c1a050efb3d6a474e90f68b05ba39a42f001ddf87dbe" + }, + { + "bytes": 21052467, + "container_path": "/opt/pillow", + "file_count": 238, + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\derived\\perception-p0-env-v1", + "id": "pillow-packages", + "includes": [ + "PIL", + "pillow-12.3.0.dist-info", + "pillow.libs" + ], + "tree_sha256": "e8ad3f69443be6c9f6233703aedf793fba40a7b54c1f7845702cce7e47699711" + } + ], + "host": { + "node": "DESKTOP-OPJ8J04", + "worker_id": "worker-006" + }, + "inputs": { + "camera_index": { + "container_path": "/source/camera/index.jsonl", + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\jobs\\recorded-camera-602ac89026ed12978619801d\\input\\camera\\sensor.camera.right\\epoch-1\\index.jsonl", + "sha256": "e029815a60ad9fbfedb6169142c7449df2b119a51d1ce001f08806e04eb0be14" + }, + "camera_summary": { + "container_path": "/source/camera/summary.json", + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\jobs\\recorded-camera-602ac89026ed12978619801d\\input\\camera\\sensor.camera.right\\epoch-1\\summary.json", + "sha256": "b280f40b198aad5d5335819107fb1405ad65d5695d187c61a2c027ad853a3181" + }, + "source_pack": { + "container_path": "/source/pack/lidar-pack.npz", + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\derived\\e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b\\lidar-pack.npz", + "sha256": "0685d24219d8236caf8b7f1685e93f6d6b59e7fd015a768d88a92bbe8b154944" + }, + "source_pack_manifest": { + "container_path": "/source/pack/manifest.json", + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\derived\\e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b\\manifest.json", + "sha256": "7e4af82eb7dfd47e242dc5207d55456cd0a5ab4c6780c584d4685ef854a73bf5" + }, + "valid_fov_mask": { + "container_path": "/source/mask/mask.png", + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\inputs\\e2\\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2\\mask.png", + "sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63" + }, + "video": { + "container_path": "/source/video/right.mp4", + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\experiments\\e46e\\inputs\\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4", + "sha256": "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8" + }, + "yolox_config": { + "container_path": null, + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\models\\yolox_s\\config.pbtxt", + "sha256": "5795c737a7935a655961b069e8404d336d891f9762fb6dffb93956a076479604" + }, + "yolox_model": { + "container_path": null, + "host_path": "D:\\NDC_MISSIONCORE\\runtime\\models\\yolox_s\\1\\model.onnx", + "sha256": "c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063" + } + }, + "patch_id": "__PATCH_ID__", + "predecessor": { + "durable_worker": { + "container_id": "db2024d05098a6beb6b73bbf43f02c88ede586a4664b2c91182f436a42b3e3ff", + "image_id": "sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", + "name": "ndc-mission-core-perception-worker", + "runner_sha256": "86e9b25c80908a520ed483541b708cde1d1c74605e094053fcb62b610ecefb20", + "worker_package_id": "dbf55ccb75664b778a2e0f5d34284af10ec0971945bbbee67b27bc264b765b51" + }, + "triton": { + "container_id": "4232fb040062a8384809e73612baa92b343ddcb60f00c23e7091eb4909959223", + "image_id": "sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", + "name": "ndc-mission-core-triton" + } + }, + "release": { + "baseline": { + "name": "m4-recorded-realtime-baseline-v1.json", + "sha256": "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8" + }, + "wheel": { + "name": "nodedc_mission_core-0.1.0-py3-none-any.whl", + "sha256": "df756938f2c212fb6d1770c83569e5366c708d70d74e0434895a21be55a16102" + } + }, + "rollback": { + "durable_worker_action": "none", + "preserve_failed_evidence": true, + "remove_candidate_container": true, + "remove_unaccepted_release": true, + "triton_action": "none" + }, + "schema_version": "nodedc.mission-core-worker.shadow-release/v1", + "transition": "m4-detector-shadow-v1" +} diff --git a/scripts/build_m4_worker_shadow_artifact.py b/scripts/build_m4_worker_shadow_artifact.py new file mode 100644 index 0000000..7a0eaa4 --- /dev/null +++ b/scripts/build_m4_worker_shadow_artifact.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Build the deterministic, data-only Worker 006 M4 detector shadow artifact.""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import os +import re +import subprocess +import tarfile +import tempfile +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +BASELINE = REPOSITORY_ROOT / "config/perception/m4-recorded-realtime-baseline-v1.json" +DESCRIPTOR_TEMPLATE = ( + REPOSITORY_ROOT / "config/deployment/mission-core-worker-shadow-v1.template.json" +) +WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl" +PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$") +EXPECTED_BASELINE_SHA256 = "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8" +EXPECTED_WHEEL_SHA256 = "df756938f2c212fb6d1770c83569e5366c708d70d74e0434895a21be55a16102" +PAYLOAD_FILES = ( + WHEEL_NAME, + "m4-recorded-realtime-baseline-v1.json", + "mission-core-worker-shadow-v1.json", +) + + +class ArtifactBuildError(RuntimeError): + pass + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_revision() -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=REPOSITORY_ROOT, + check=True, + capture_output=True, + text=True, + ) + revision = result.stdout.strip() + if re.fullmatch(r"[a-f0-9]{40}", revision) is None: + raise ArtifactBuildError("Git revision is not a full SHA-1") + return revision + + +def build_wheel(output: Path) -> Path: + environment = os.environ.copy() + environment["SOURCE_DATE_EPOCH"] = "0" + result = subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(output)], + cwd=REPOSITORY_ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + detail = (result.stderr or result.stdout).strip() + raise ArtifactBuildError(f"wheel build failed: {detail}") + wheel = output / WHEEL_NAME + if not wheel.is_file() or wheel.is_symlink(): + raise ArtifactBuildError("expected wheel was not built") + if sha256_file(wheel) != EXPECTED_WHEEL_SHA256: + raise ArtifactBuildError("runtime wheel digest changed") + return wheel + + +def render_descriptor(patch_id: str, revision: str) -> bytes: + template = DESCRIPTOR_TEMPLATE.read_text("utf-8") + if template.count("__PATCH_ID__") != 1 or template.count("__CODE_REVISION__") != 1: + raise ArtifactBuildError("descriptor template placeholders changed") + rendered = template.replace("__PATCH_ID__", patch_id).replace("__CODE_REVISION__", revision) + document = json.loads(rendered) + if document["patch_id"] != patch_id or document["code_revision"] != revision: + raise ArtifactBuildError("descriptor identity rendering failed") + return (json.dumps(document, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode( + "utf-8" + ) + + +def _tar_info(path: Path, arcname: str) -> tarfile.TarInfo: + info = tarfile.TarInfo(arcname) + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + info.mtime = 0 + if path.is_dir(): + info.type = tarfile.DIRTYPE + info.mode = 0o755 + else: + info.type = tarfile.REGTYPE + info.mode = 0o644 + info.size = path.stat().st_size + return info + + +def write_canonical_archive(stage: Path, target: Path) -> None: + members = [stage / "manifest.env", stage / "files.txt", stage / "payload"] + members.extend(sorted((stage / "payload").rglob("*"))) + target.parent.mkdir(parents=True, exist_ok=True) + with ( + target.open("wb") as raw, + gzip.GzipFile(filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=0) as gz, + tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as archive, + ): + for path in members: + arcname = path.relative_to(stage).as_posix() + info = _tar_info(path, arcname) + if path.is_file(): + with path.open("rb") as source: + archive.addfile(info, source) + else: + archive.addfile(info, io.BytesIO()) + + +def build_artifact(patch_id: str, output_directory: Path) -> dict[str, object]: + if PATCH_ID.fullmatch(patch_id) is None: + raise ArtifactBuildError("patch id is invalid") + if sha256_file(BASELINE) != EXPECTED_BASELINE_SHA256: + raise ArtifactBuildError("baseline digest changed") + revision = git_revision() + with tempfile.TemporaryDirectory(prefix="mission-core-worker-shadow-") as directory: + stage = Path(directory) + payload = stage / "payload" + payload.mkdir() + wheel = build_wheel(stage / "wheel") + (payload / WHEEL_NAME).write_bytes(wheel.read_bytes()) + (payload / "m4-recorded-realtime-baseline-v1.json").write_bytes(BASELINE.read_bytes()) + descriptor_bytes = render_descriptor(patch_id, revision) + (payload / "mission-core-worker-shadow-v1.json").write_bytes(descriptor_bytes) + (stage / "manifest.env").write_text( + f"id={patch_id}\ncomponent=mission-core-worker\ntype=shadow-release\n", + "utf-8", + ) + (stage / "files.txt").write_text("\n".join(PAYLOAD_FILES) + "\n", "utf-8") + target = output_directory.resolve() / f"nodedc-mission-core-worker-{patch_id}.tgz" + write_canonical_archive(stage, target) + return { + "ok": True, + "patch_id": patch_id, + "component": "mission-core-worker", + "type": "shadow-release", + "artifact": str(target), + "sha256": sha256_file(target), + "code_revision": revision, + "wheel_sha256": EXPECTED_WHEEL_SHA256, + "baseline_sha256": EXPECTED_BASELINE_SHA256, + "payload_files": list(PAYLOAD_FILES), + "transition": "m4-detector-shadow-v1", + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("patch_id") + parser.add_argument( + "--output-directory", + type=Path, + default=REPOSITORY_ROOT / ".runtime/deploy-artifacts", + ) + arguments = parser.parse_args() + try: + result = build_artifact(arguments.patch_id, arguments.output_directory) + except (ArtifactBuildError, OSError, subprocess.SubprocessError, json.JSONDecodeError) as exc: + parser.error(str(exc)) + print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_m4_worker_shadow_artifact.py b/tests/test_m4_worker_shadow_artifact.py new file mode 100644 index 0000000..82438aa --- /dev/null +++ b/tests/test_m4_worker_shadow_artifact.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import tarfile +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +BUILDER_PATH = REPOSITORY_ROOT / "scripts/build_m4_worker_shadow_artifact.py" +SPEC = importlib.util.spec_from_file_location("m4_worker_shadow_builder", BUILDER_PATH) +assert SPEC is not None and SPEC.loader is not None +BUILDER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(BUILDER) + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _regular_files(archive: tarfile.TarFile) -> dict[str, bytes]: + result: dict[str, bytes] = {} + for member in archive.getmembers(): + if not member.isfile(): + continue + extracted = archive.extractfile(member) + assert extracted is not None + result[member.name] = extracted.read() + return result + + +def test_worker_shadow_artifact_is_deterministic_narrow_and_data_only( + tmp_path: Path, +) -> None: + patch_id = "mission-core-m4-detector-shadow-unit-001" + first = BUILDER.build_artifact(patch_id, tmp_path / "first") + second = BUILDER.build_artifact(patch_id, tmp_path / "second") + first_bytes = Path(first["artifact"]).read_bytes() + second_bytes = Path(second["artifact"]).read_bytes() + + assert first_bytes == second_bytes + assert first["sha256"] == _sha256(first_bytes) + assert first["sha256"] == second["sha256"] + assert first["payload_files"] == list(BUILDER.PAYLOAD_FILES) + + with tarfile.open(first["artifact"], "r:gz") as archive: + members = archive.getmembers() + names = [member.name for member in members] + regular = _regular_files(archive) + + assert names == [ + "manifest.env", + "files.txt", + "payload", + "payload/m4-recorded-realtime-baseline-v1.json", + "payload/mission-core-worker-shadow-v1.json", + f"payload/{BUILDER.WHEEL_NAME}", + ] + assert ( + regular["manifest.env"] + == (f"id={patch_id}\ncomponent=mission-core-worker\ntype=shadow-release\n").encode() + ) + assert regular["files.txt"].decode().splitlines() == list(BUILDER.PAYLOAD_FILES) + assert _sha256(regular[f"payload/{BUILDER.WHEEL_NAME}"]) == (BUILDER.EXPECTED_WHEEL_SHA256) + assert _sha256(regular["payload/m4-recorded-realtime-baseline-v1.json"]) == ( + BUILDER.EXPECTED_BASELINE_SHA256 + ) + descriptor = json.loads(regular["payload/mission-core-worker-shadow-v1.json"]) + assert descriptor["patch_id"] == patch_id + assert descriptor["code_revision"] == first["code_revision"] + assert descriptor["container"]["public_ports"] is False + assert descriptor["rollback"] == { + "durable_worker_action": "none", + "preserve_failed_evidence": True, + "remove_candidate_container": True, + "remove_unaccepted_release": True, + "triton_action": "none", + } + all_bytes = b"\n".join(regular.values()) + assert b"PRIVATE KEY" not in all_bytes + assert b"password=" not in all_bytes.lower() + assert not any( + Path(name).name.startswith(".env") + or "/secrets/" in name + or "/runtime/" in name + or "/recordings/" in name + for name in names + ) + + +def test_worker_shadow_descriptor_pins_external_inputs_and_preserves_e15() -> None: + rendered = BUILDER.render_descriptor( + "mission-core-m4-detector-shadow-unit-002", + "c" * 40, + ) + descriptor = json.loads(rendered) + + assert descriptor["schema_version"] == ("nodedc.mission-core-worker.shadow-release/v1") + assert descriptor["transition"] == "m4-detector-shadow-v1" + assert descriptor["host"] == { + "node": "DESKTOP-OPJ8J04", + "worker_id": "worker-006", + } + assert set(descriptor["inputs"]) == { + "camera_index", + "camera_summary", + "source_pack", + "source_pack_manifest", + "valid_fov_mask", + "video", + "yolox_config", + "yolox_model", + } + assert all(len(value["sha256"]) == 64 for value in descriptor["inputs"].values()) + assert [item["id"] for item in descriptor["dependencies"]] == [ + "pyav-media", + "opencv-packages", + "pillow-packages", + ] + assert descriptor["predecessor"]["durable_worker"]["name"] == ( + "ndc-mission-core-perception-worker" + ) + assert descriptor["predecessor"]["triton"]["name"] == ("ndc-mission-core-triton") + assert descriptor["acceptance"] == { + "class_routing_used": False, + "expected_frames": 4489, + "failed_frames": 0, + "minimum_end_to_end_fps": 10.004, + }