build(worker): package M4.8R3 shadow release
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the deterministic M4.8R3 occupied-only Worker 006 shadow release."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl"
|
||||
POWERSHELL_RUNNER = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments/perception/worker/Invoke-M48NNativeReferenceGraph.ps1"
|
||||
)
|
||||
PYTHON_RUNNER = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments/perception/run_m48s_reference_graph_shadow_worker.py"
|
||||
)
|
||||
NATIVE_CONFIG = (
|
||||
REPOSITORY_ROOT / "experiments/perception/worker/rf_detr_large_native_kb4_config.pbtxt"
|
||||
)
|
||||
CONFIG_PATHS = (
|
||||
Path("config/perception/m48r3-native-low-step-reference-graph-shadow-v1.json"),
|
||||
Path("config/perception/m48r3-additive-low-step-occupancy-v1.json"),
|
||||
Path("config/perception/m4-recorded-realtime-baseline-v1.json"),
|
||||
Path("config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json"),
|
||||
Path("config/perception/m4-geometry-association-v1.json"),
|
||||
Path("config/perception/m4-temporal-motion-v1.json"),
|
||||
Path("config/perception/m4-rolling-local-map-v1.json"),
|
||||
Path("config/perception/m4-replay-threat-v3.json"),
|
||||
)
|
||||
PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$")
|
||||
|
||||
|
||||
class ArtifactBuildError(RuntimeError):
|
||||
"""The M4.8R3 release cannot be built from the declared source."""
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def git_revision(*, require_clean: bool) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
revision = result.stdout.strip()
|
||||
if re.fullmatch(r"[a-f0-9]{40}", revision) is None:
|
||||
raise ArtifactBuildError("Git revision is not a full SHA-1")
|
||||
if require_clean:
|
||||
status = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if status.stdout.strip():
|
||||
raise ArtifactBuildError("Worker shadow artifact requires a clean worktree")
|
||||
return revision
|
||||
|
||||
|
||||
def build_wheel(output: Path) -> Path:
|
||||
environment = os.environ.copy()
|
||||
environment["SOURCE_DATE_EPOCH"] = "0"
|
||||
result = subprocess.run(
|
||||
["uv", "build", "--wheel", "--out-dir", str(output)],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
env=environment,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = (result.stderr or result.stdout).strip()
|
||||
raise ArtifactBuildError(f"wheel build failed: {detail}")
|
||||
wheel = output / WHEEL_NAME
|
||||
if not wheel.is_file() or wheel.is_symlink():
|
||||
raise ArtifactBuildError("expected Worker wheel was not built")
|
||||
return wheel
|
||||
|
||||
|
||||
def _tar_info(path: Path, arcname: str) -> tarfile.TarInfo:
|
||||
info = tarfile.TarInfo(arcname)
|
||||
info.uid = 0
|
||||
info.gid = 0
|
||||
info.uname = "root"
|
||||
info.gname = "root"
|
||||
info.mtime = 0
|
||||
if path.is_dir():
|
||||
info.type = tarfile.DIRTYPE
|
||||
info.mode = 0o755
|
||||
else:
|
||||
info.type = tarfile.REGTYPE
|
||||
info.mode = 0o644
|
||||
info.size = path.stat().st_size
|
||||
return info
|
||||
|
||||
|
||||
def write_canonical_archive(stage: Path, target: Path) -> None:
|
||||
members = [stage / "manifest.env", stage / "files.txt", stage / "payload"]
|
||||
members.extend(sorted((stage / "payload").rglob("*")))
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with (
|
||||
target.open("wb") as raw,
|
||||
gzip.GzipFile(filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=0) as gz,
|
||||
tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as archive,
|
||||
):
|
||||
for path in members:
|
||||
info = _tar_info(path, path.relative_to(stage).as_posix())
|
||||
if path.is_file():
|
||||
with path.open("rb") as stream:
|
||||
archive.addfile(info, stream)
|
||||
else:
|
||||
archive.addfile(info, io.BytesIO())
|
||||
|
||||
|
||||
def build_artifact(
|
||||
patch_id: str,
|
||||
output_directory: Path,
|
||||
*,
|
||||
revision: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
if PATCH_ID.fullmatch(patch_id) is None:
|
||||
raise ArtifactBuildError("patch id is invalid")
|
||||
selected_revision = revision or git_revision(require_clean=True)
|
||||
if re.fullmatch(r"[a-f0-9]{40}", selected_revision) is None:
|
||||
raise ArtifactBuildError("artifact revision is invalid")
|
||||
sources = (
|
||||
POWERSHELL_RUNNER,
|
||||
PYTHON_RUNNER,
|
||||
NATIVE_CONFIG,
|
||||
*(REPOSITORY_ROOT / relative for relative in CONFIG_PATHS),
|
||||
)
|
||||
if any(path.is_symlink() or not path.is_file() for path in sources):
|
||||
raise ArtifactBuildError("Worker release input is not a regular file")
|
||||
with tempfile.TemporaryDirectory(prefix="mission-core-m48r3-worker-") as directory:
|
||||
stage = Path(directory)
|
||||
payload = stage / "payload"
|
||||
payload.mkdir()
|
||||
wheel = build_wheel(stage / "wheel")
|
||||
copied: list[Path] = []
|
||||
for source in sources:
|
||||
destination = payload / source.name
|
||||
destination.write_bytes(source.read_bytes())
|
||||
copied.append(destination)
|
||||
wheel_destination = payload / WHEEL_NAME
|
||||
wheel_destination.write_bytes(wheel.read_bytes())
|
||||
copied.append(wheel_destination)
|
||||
release = {
|
||||
"schema_version": "missioncore.m48r3-worker-shadow-release/v1",
|
||||
"patch_id": patch_id,
|
||||
"transition": "m48r3-additive-low-step-isolated-shadow/v1",
|
||||
"code_revision": selected_revision,
|
||||
"worker_id": "worker-006",
|
||||
"graph_id": "reference-perception-graph/v2",
|
||||
"detector_provider_id": (
|
||||
"triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0"
|
||||
),
|
||||
"geometry_provider_id": "ravnoves00-additive-low-step-geometry/v1",
|
||||
"expected_frames": 4489,
|
||||
"requested_source_rate_hz": 12.0,
|
||||
"candidate_accepted": False,
|
||||
"production_accepted": False,
|
||||
"durable_worker_action": "none",
|
||||
"canonical_triton_action": "none",
|
||||
"files": {
|
||||
path.name: {"sha256": sha256_file(path), "bytes": path.stat().st_size}
|
||||
for path in sorted(copied)
|
||||
},
|
||||
}
|
||||
release_path = payload / "release.json"
|
||||
release_path.write_text(
|
||||
json.dumps(release, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
"utf-8",
|
||||
)
|
||||
payload_files = sorted((*release["files"], release_path.name))
|
||||
(stage / "manifest.env").write_text(
|
||||
f"id={patch_id}\ncomponent=mission-core-worker\ntype=shadow-release\n",
|
||||
"utf-8",
|
||||
)
|
||||
(stage / "files.txt").write_text("\n".join(payload_files) + "\n", "utf-8")
|
||||
target = output_directory.resolve() / f"nodedc-{patch_id}.tgz"
|
||||
write_canonical_archive(stage, target)
|
||||
return {
|
||||
"ok": True,
|
||||
"patch_id": patch_id,
|
||||
"artifact": str(target),
|
||||
"sha256": sha256_file(target),
|
||||
"code_revision": selected_revision,
|
||||
"wheel_sha256": release["files"][WHEEL_NAME]["sha256"],
|
||||
"payload_files": payload_files,
|
||||
"transition": release["transition"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("patch_id")
|
||||
parser.add_argument(
|
||||
"--output-directory",
|
||||
type=Path,
|
||||
default=REPOSITORY_ROOT / ".runtime/worker-artifacts",
|
||||
)
|
||||
arguments = parser.parse_args()
|
||||
try:
|
||||
result = build_artifact(arguments.patch_id, arguments.output_directory)
|
||||
except (ArtifactBuildError, OSError, subprocess.SubprocessError) as exc:
|
||||
parser.error(str(exc))
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
BUILDER_PATH = REPOSITORY_ROOT / "scripts/build_m48r3_worker_shadow_artifact.py"
|
||||
SPEC = importlib.util.spec_from_file_location("m48r3_worker_shadow_builder", BUILDER_PATH)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
BUILDER = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(BUILDER)
|
||||
|
||||
|
||||
def _sha256(value: bytes) -> str:
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
|
||||
def _regular_files(archive: tarfile.TarFile) -> dict[str, bytes]:
|
||||
result: dict[str, bytes] = {}
|
||||
for member in archive.getmembers():
|
||||
if not member.isfile():
|
||||
continue
|
||||
stream = archive.extractfile(member)
|
||||
assert stream is not None
|
||||
result[member.name] = stream.read()
|
||||
return result
|
||||
|
||||
|
||||
def test_m48r3_worker_artifact_is_deterministic_and_bounded(tmp_path: Path) -> None:
|
||||
patch_id = "mission-core-m48r3-low-step-unit-001"
|
||||
revision = "a" * 40
|
||||
first = BUILDER.build_artifact(patch_id, tmp_path / "first", revision=revision)
|
||||
second = BUILDER.build_artifact(patch_id, tmp_path / "second", revision=revision)
|
||||
|
||||
first_bytes = Path(first["artifact"]).read_bytes()
|
||||
assert first_bytes == Path(second["artifact"]).read_bytes()
|
||||
assert first["sha256"] == _sha256(first_bytes)
|
||||
with tarfile.open(first["artifact"], "r:gz") as archive:
|
||||
members = archive.getmembers()
|
||||
regular = _regular_files(archive)
|
||||
assert all(not member.issym() and not member.islnk() for member in members)
|
||||
assert set(regular) == {
|
||||
"manifest.env",
|
||||
"files.txt",
|
||||
*(f"payload/{name}" for name in first["payload_files"]),
|
||||
}
|
||||
assert regular["files.txt"].decode().splitlines() == first["payload_files"]
|
||||
release = json.loads(regular["payload/release.json"])
|
||||
assert release["code_revision"] == revision
|
||||
assert release["geometry_provider_id"] == (
|
||||
"ravnoves00-additive-low-step-geometry/v1"
|
||||
)
|
||||
assert release["candidate_accepted"] is False
|
||||
assert release["production_accepted"] is False
|
||||
assert release["durable_worker_action"] == "none"
|
||||
assert release["canonical_triton_action"] == "none"
|
||||
serialized = json.dumps(release).lower()
|
||||
assert "private key" not in serialized
|
||||
assert "password=" not in serialized
|
||||
Reference in New Issue
Block a user