feat(lab): add sealed local component runners

This commit is contained in:
DCCONSTRUCTIONS
2026-08-31 18:59:47 +03:00
parent 3744d79ad3
commit f41428f907
2 changed files with 1862 additions and 0 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,531 @@
from __future__ import annotations
import json
from dataclasses import dataclass
from pathlib import Path
from typing import cast
import httpx
import pytest
from k1link.observatory import portable_lab_v1_local_runners as local_runners
from k1link.observatory.portable_lab_v1_executor import (
PortableLabV1MaterializedSource,
PortableLabV1OrchestrationPlan,
PortableLabV1SourceInput,
)
from k1link.observatory.portable_result_contract import canonical_json
from k1link.observatory.portable_run_definitions import canonical_sha256
EOMT_IMAGE_SHA256 = "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
DDRNET_IMAGE_SHA256 = "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
DEFINITION_SHA256 = "a" * 64
RELEASE_SHA256 = "b" * 64
@dataclass(frozen=True)
class _Plan:
setup_id: str
definition_id: str
definition_version: int
definition_sha256: str
release_candidate_sha256: str
source_input: PortableLabV1SourceInput
effective_ddrnet_config: dict[str, object]
effective_ddrnet_config_sha256: str
plan_sha256: str
def require_executable(self) -> None:
return None
def _assets(
component: local_runners.PortableLabV1Component,
) -> tuple[local_runners.PortableLabV1HostAsset, ...]:
identities = local_runners._EXPECTED_ASSET_IDENTITIES[component]
tree_assets = {"eomt-dependency-set", "k1-valid-fov-identity"}
return tuple(
local_runners.PortableLabV1HostAsset(
asset_id=asset_id,
host_path=f"D:\\NDC_MISSIONCORE\\assets\\{asset_id}",
container_path=f"/opt/nodedc/assets/{asset_id}",
kind="tree" if asset_id in tree_assets else "file",
verification=("identity-sha256" if asset_id in tree_assets else "sha256"),
identity_sha256=identity,
byte_length=local_runners._EXPECTED_ASSET_LENGTHS.get(asset_id),
)
for asset_id, identity in sorted(identities.items())
)
def _component(
component: local_runners.PortableLabV1Component,
) -> local_runners.PortableLabV1ComponentInstallation:
return local_runners.PortableLabV1ComponentInstallation.seal(
component=component,
image_sha256=(EOMT_IMAGE_SHA256 if component == "eomt" else DDRNET_IMAGE_SHA256),
entrypoint=("/opt/nodedc/bin/portable-lab-v1-agent",),
command=(f"run-{component}",),
assets=_assets(component),
timeout_seconds=3600.0,
memory_bytes=16 * 1024**3,
nano_cpus=4_000_000_000,
)
def _installation(tmp_path: Path) -> local_runners.PortableLabV1RunnerInstallation:
return local_runners.PortableLabV1RunnerInstallation.seal(
definition_sha256=DEFINITION_SHA256,
release_candidate_sha256=RELEASE_SHA256,
work_root=local_runners.PortableLabV1WorkRootBinding(
controller_root=tmp_path,
engine_host_root="D:\\NDC_MISSIONCORE\\runtime\\observatory-worker",
),
eomt=_component("eomt"),
ddrnet=_component("ddrnet"),
)
def _source(tmp_path: Path) -> PortableLabV1MaterializedSource:
root = tmp_path / "source"
camera = root / "camera-job"
camera.mkdir(parents=True)
descriptor = PortableLabV1SourceInput(
observatory_job_id=f"observatory-run-{'1' * 32}",
observatory_request_sha256="2" * 64,
observatory_identity_sha256="3" * 64,
source_session_id="20260831T083000Z_viewer_live",
source_catalog_sha256="4" * 64,
source_bundle_sha256="5" * 64,
source_capability_manifest_sha256="6" * 64,
source_adapter_sha256="7" * 64,
camera_job_id=f"recorded-camera-{'8' * 24}",
camera_input_sha256="9" * 64,
camera_source_id="sensor.camera.right",
codec_epoch=1,
input_byte_length=64,
frame_count=2,
timeline_start_seconds=1.0,
timeline_end_seconds=3.0,
camera_generation_sha256="c" * 64,
calibration_sha256="d" * 64,
)
return PortableLabV1MaterializedSource(
root=root,
camera_job_root=camera,
descriptor=descriptor,
)
def _plan(source: PortableLabV1MaterializedSource) -> PortableLabV1OrchestrationPlan:
effective: dict[str, object] = {
"schema_version": "missioncore.lab-v1-goose-vegetation-benchmark/v1",
"ravnoves": {
"source_id": (
"portable-k1/20260831T083000Z_viewer_live/"
f"sensor.camera.right@{source.descriptor.camera_input_sha256}"
),
"source_sha256": source.descriptor.camera_input_sha256,
},
}
plan = _Plan(
setup_id="lab-v1-eomt-ddrnet-portable-v1",
definition_id="lab-v1-eomt-ddrnet-portable",
definition_version=2,
definition_sha256=DEFINITION_SHA256,
release_candidate_sha256=RELEASE_SHA256,
source_input=source.descriptor,
effective_ddrnet_config=effective,
effective_ddrnet_config_sha256=canonical_sha256(effective),
plan_sha256="e" * 64,
)
return cast(PortableLabV1OrchestrationPlan, plan)
def _docker_launch(tmp_path: Path) -> local_runners.PortableLabV1DockerLaunch:
output = tmp_path / "output"
output.mkdir()
return local_runners.PortableLabV1DockerLaunch(
component="eomt",
image_sha256=EOMT_IMAGE_SHA256,
entrypoint=("/opt/nodedc/bin/portable-lab-v1-agent",),
command=("run-eomt", "--request", "/run/nodedc/request.json"),
mounts=(
local_runners.PortableLabV1DockerMount(
host_path="D:\\NDC_MISSIONCORE\\runtime\\output",
container_path="/run/nodedc/output",
read_only=False,
),
local_runners.PortableLabV1DockerMount(
host_path="D:\\NDC_MISSIONCORE\\runtime\\request.json",
container_path="/run/nodedc/request.json",
read_only=True,
),
),
labels={
"com.nodedc.authority": "observation-only",
"com.nodedc.component": "eomt",
"com.nodedc.definition-sha256": DEFINITION_SHA256,
"com.nodedc.managed-by": "mission-core-worker",
"com.nodedc.request-sha256": "c" * 64,
"com.nodedc.product": "mission-core",
"com.nodedc.release-candidate-sha256": RELEASE_SHA256,
"com.nodedc.stack": "observatory",
},
timeout_seconds=60.0,
memory_bytes=8 * 1024**3,
nano_cpus=2_000_000_000,
name_token="0123456789abcdef",
local_output_root=output,
)
def test_docker_engine_launcher_uses_exact_hardened_sibling_contract(
tmp_path: Path,
) -> None:
requests: list[httpx.Request] = []
create_document: dict[str, object] = {}
container_id = "f" * 64
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
if request.url.path.startswith("/v1.47/images/"):
return httpx.Response(200, json={"Id": f"sha256:{EOMT_IMAGE_SHA256}"})
if request.url.path == "/v1.47/containers/create":
create_document.update(json.loads(request.content))
return httpx.Response(201, json={"Id": container_id, "Warnings": None})
if request.url.path.endswith("/start"):
return httpx.Response(204)
if request.url.path.endswith("/wait"):
return httpx.Response(200, json={"StatusCode": 0, "Error": None})
if request.method == "DELETE":
return httpx.Response(204)
raise AssertionError(f"unexpected request: {request.method} {request.url}")
launcher = local_runners.DockerEnginePortableLabV1Launcher(
transport_factory=lambda: httpx.MockTransport(handler)
)
launcher(_docker_launch(tmp_path))
assert [request.method for request in requests] == [
"GET",
"POST",
"POST",
"POST",
"DELETE",
]
assert create_document["Image"] == f"sha256:{EOMT_IMAGE_SHA256}"
assert create_document["NetworkDisabled"] is True
host = cast(dict[str, object], create_document["HostConfig"])
assert host["NetworkMode"] == "none"
assert host["ReadonlyRootfs"] is True
assert host["CapDrop"] == ["ALL"]
assert host["SecurityOpt"] == ["no-new-privileges:true"]
assert host["Privileged"] is False
assert host["DeviceRequests"] == [{"Driver": "nvidia", "Count": 1, "Capabilities": [["gpu"]]}]
mounts = cast(list[dict[str, object]], host["Mounts"])
assert sum(not cast(bool, mount["ReadOnly"]) for mount in mounts) == 1
assert next(mount for mount in mounts if not mount["ReadOnly"])["Target"] == (
"/run/nodedc/output"
)
def test_docker_engine_launcher_fails_closed_and_cleans_nonzero_container(
tmp_path: Path,
) -> None:
requests: list[httpx.Request] = []
container_id = "a" * 64
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
if request.url.path.startswith("/v1.47/images/"):
return httpx.Response(200, json={"Id": f"sha256:{EOMT_IMAGE_SHA256}"})
if request.url.path == "/v1.47/containers/create":
return httpx.Response(201, json={"Id": container_id, "Warnings": None})
if request.url.path.endswith("/start"):
return httpx.Response(204)
if request.url.path.endswith("/wait"):
return httpx.Response(200, json={"StatusCode": 17, "Error": None})
if request.url.path.endswith("/logs"):
return httpx.Response(200, content=b"sensitive worker error")
if request.method == "DELETE":
return httpx.Response(204)
raise AssertionError(f"unexpected request: {request.method} {request.url}")
launcher = local_runners.DockerEnginePortableLabV1Launcher(
transport_factory=lambda: httpx.MockTransport(handler)
)
with pytest.raises(
local_runners.PortableLabV1LocalRunnerError,
match=r"exited with status 17; logs=[a-f0-9]{64}:22",
):
launcher(_docker_launch(tmp_path))
assert requests[-1].method == "DELETE"
assert "sensitive worker error" not in str(requests)
def test_work_root_binding_rejects_escape_and_non_d_host_root(tmp_path: Path) -> None:
binding = local_runners.PortableLabV1WorkRootBinding(
controller_root=tmp_path,
engine_host_root="D:\\NDC_MISSIONCORE\\runtime\\observatory-worker",
)
admitted = tmp_path / "jobs" / "job-001"
admitted.mkdir(parents=True)
assert binding.engine_path(admitted, label="job") == (
"D:\\NDC_MISSIONCORE\\runtime\\observatory-worker\\jobs\\job-001"
)
outside = tmp_path.parent / "outside"
outside.mkdir(exist_ok=True)
with pytest.raises(
local_runners.PortableLabV1LocalRunnerError,
match="outside the installed work root",
):
binding.engine_path(outside, label="outside")
windows_escape = tmp_path / r"foo\..\outside"
windows_escape.mkdir()
with pytest.raises(
local_runners.PortableLabV1LocalRunnerError,
match="Windows-unsafe path component",
):
binding.engine_path(windows_escape, label="Windows escape")
with pytest.raises(
local_runners.PortableLabV1LocalRunnerError,
match="absolute D: path",
):
local_runners.PortableLabV1WorkRootBinding(
controller_root=tmp_path,
engine_host_root="C:\\temp",
)
def test_docker_engine_launcher_bounds_control_plane_responses(tmp_path: Path) -> None:
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(
200,
headers={"content-length": str(local_runners._MAX_ENGINE_ERROR_BYTES + 1)},
)
launcher = local_runners.DockerEnginePortableLabV1Launcher(
transport_factory=lambda: httpx.MockTransport(handler)
)
with pytest.raises(
local_runners.PortableLabV1LocalRunnerError,
match="Docker Engine response is too large",
):
launcher(_docker_launch(tmp_path))
assert [request.method for request in requests] == ["GET"]
def test_docker_engine_launcher_retries_cleanup_after_create_warning(
tmp_path: Path,
) -> None:
requests: list[httpx.Request] = []
container_id = "b" * 64
cleanup_attempts = 0
def handler(request: httpx.Request) -> httpx.Response:
nonlocal cleanup_attempts
requests.append(request)
if request.url.path.startswith("/v1.47/images/"):
return httpx.Response(200, json={"Id": f"sha256:{EOMT_IMAGE_SHA256}"})
if request.url.path == "/v1.47/containers/create":
return httpx.Response(
201,
json={"Id": container_id, "Warnings": ["engine warning"]},
)
if request.method == "DELETE":
cleanup_attempts += 1
return httpx.Response(500 if cleanup_attempts == 1 else 204)
raise AssertionError(f"unexpected request: {request.method} {request.url}")
launcher = local_runners.DockerEnginePortableLabV1Launcher(
transport_factory=lambda: httpx.MockTransport(handler)
)
with pytest.raises(
local_runners.PortableLabV1LocalRunnerError,
match="Docker container creation returned warnings",
):
launcher(_docker_launch(tmp_path))
assert cleanup_attempts == 2
assert [request.method for request in requests] == ["GET", "POST", "DELETE", "DELETE"]
def test_installed_runners_bind_plan_config_and_component_outputs(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
local_runners,
"validate_camera_compute_job",
lambda _path: object(),
)
installation = _installation(tmp_path)
source = _source(tmp_path)
plan = _plan(source)
launches: list[local_runners.PortableLabV1DockerLaunch] = []
def launcher(launch: local_runners.PortableLabV1DockerLaunch) -> None:
launches.append(launch)
if launch.component == "eomt":
frames_root = launch.local_output_root / "source-frames"
frames_root.mkdir()
for sequence in range(1, plan.source_input.frame_count + 1):
(frames_root / f"frame-{sequence:06d}.png").write_bytes(
f"frame-{sequence}".encode()
)
document: dict[str, object] = {
"schema_version": "missioncore.recorded-perception-result/v2",
"job_id": plan.source_input.camera_job_id,
"input_sha256": plan.source_input.camera_input_sha256,
"session_id": plan.source_input.source_session_id,
"source_id": plan.source_input.camera_source_id,
"codec_epoch": plan.source_input.codec_epoch,
"frames_processed": plan.source_input.frame_count,
"ground_truth": False,
"identity": {
"configuration": {"pipeline": "recorded-semantic-eomt-fisheye-mask/v1"}
},
}
else:
effective_source = cast(
dict[str, object],
plan.effective_ddrnet_config["ravnoves"],
)
document = {
"schema_version": "missioncore.lab-v1-goose-vegetation-run/v1",
"mode": "ravnoves-video",
"candidate": {
"candidate_id": "goose-ddrnet-class-512",
"candidate_key": "ddrnet",
"checkpoint_sha256": (
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
),
},
"source": {
"source_id": effective_source["source_id"],
"input_count": plan.source_input.frame_count,
"mapping_sha256": (
"88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f"
),
},
"provenance": {"config_sha256": plan.effective_ddrnet_config_sha256},
}
(launch.local_output_root / "result.json").write_bytes(canonical_json(document))
installed = local_runners.compose_portable_lab_v1_installed_runners(
installation=installation,
launcher=launcher,
)
eomt_root = tmp_path / "eomt-result"
installed.eomt(source=source, plan=plan, output_root=eomt_root)
effective_path = tmp_path / "effective-ddrnet-config.json"
effective_path.write_bytes(canonical_json(plan.effective_ddrnet_config))
ddrnet_root = tmp_path / "ddrnet-result"
installed.ddrnet(
source=source,
plan=plan,
effective_config_path=effective_path,
eomt_result_root=eomt_root,
output_root=ddrnet_root,
)
assert [launch.component for launch in launches] == ["eomt", "ddrnet"]
assert (eomt_root / "result.json").is_file()
assert (ddrnet_root / "result.json").is_file()
assert all(
launch.command[-2:] == ("--request", "/run/nodedc/request.json") for launch in launches
)
assert all(sum(not mount.read_only for mount in launch.mounts) == 1 for launch in launches)
assert not tuple(tmp_path.glob(".lab-v1-*-invocation-*"))
def test_ddrnet_runner_rejects_mutated_effective_config_before_launch(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
local_runners,
"validate_camera_compute_job",
lambda _path: object(),
)
installation = _installation(tmp_path)
source = _source(tmp_path)
plan = _plan(source)
eomt_root = tmp_path / "eomt-result"
eomt_root.mkdir()
frames_root = eomt_root / "source-frames"
frames_root.mkdir()
for sequence in range(1, plan.source_input.frame_count + 1):
(frames_root / f"frame-{sequence:06d}.png").write_bytes(f"frame-{sequence}".encode())
(eomt_root / "result.json").write_bytes(
canonical_json(
{
"schema_version": "missioncore.recorded-perception-result/v2",
"job_id": plan.source_input.camera_job_id,
"input_sha256": plan.source_input.camera_input_sha256,
"session_id": plan.source_input.source_session_id,
"source_id": plan.source_input.camera_source_id,
"codec_epoch": plan.source_input.codec_epoch,
"frames_processed": plan.source_input.frame_count,
"ground_truth": False,
"identity": {
"configuration": {"pipeline": "recorded-semantic-eomt-fisheye-mask/v1"}
},
}
)
)
effective_path = tmp_path / "effective-ddrnet-config.json"
effective_path.write_text("{}", encoding="utf-8")
called = False
def launcher(_launch: local_runners.PortableLabV1DockerLaunch) -> None:
nonlocal called
called = True
runner = local_runners.InstalledPortableLabV1DdrnetRunner(
installation=installation,
launcher=cast(local_runners.PortableLabV1ContainerLauncher, launcher),
)
with pytest.raises(
local_runners.PortableLabV1LocalRunnerError,
match="effective DDRNet config changed",
):
runner(
source=source,
plan=plan,
effective_config_path=effective_path,
eomt_result_root=eomt_root,
output_root=tmp_path / "ddrnet-result",
)
assert called is False
def test_component_installation_rejects_wrong_worker006_asset_identity() -> None:
assets = list(_assets("ddrnet"))
assets[0] = local_runners.PortableLabV1HostAsset(
asset_id=assets[0].asset_id,
host_path=assets[0].host_path,
container_path=assets[0].container_path,
kind=assets[0].kind,
verification=assets[0].verification,
identity_sha256="0" * 64,
)
with pytest.raises(
local_runners.PortableLabV1LocalRunnerError,
match="installed asset identity changed",
):
local_runners.PortableLabV1ComponentInstallation.seal(
component="ddrnet",
image_sha256=DDRNET_IMAGE_SHA256,
entrypoint=("/opt/nodedc/bin/portable-lab-v1-agent",),
command=("run-ddrnet",),
assets=tuple(assets),
timeout_seconds=3600.0,
memory_bytes=16 * 1024**3,
nano_cpus=4_000_000_000,
)