feat(observatory): ship modular AI inference labs

This commit is contained in:
DCCONSTRUCTIONS
2026-09-04 17:59:05 +03:00
parent eff60e490a
commit cada687173
145 changed files with 17651 additions and 1667 deletions
@@ -0,0 +1,29 @@
FROM ndc/mission-core-installed-lab-v1-eomt-step:ee0efdd9af72
ARG NODEDC_SHARED_SHA256
ARG NODEDC_EOMT_SHA256
ARG NODEDC_MODULE_SHA256
ARG NODEDC_REVISION
COPY portable_lab_v1_component_adapter.py /opt/nodedc/adapter/portable_lab_v1_component_adapter.py
COPY run_portable_lab_v1_eomt_component.py /opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py
COPY run_portable_camera_source_component.py /opt/nodedc/adapter/run_portable_camera_source_component.py
RUN test "$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py | cut -d' ' -f1)" = "${NODEDC_SHARED_SHA256}" \
&& test "$(sha256sum /opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py | cut -d' ' -f1)" = "${NODEDC_EOMT_SHA256}" \
&& test "$(sha256sum /opt/nodedc/adapter/run_portable_camera_source_component.py | cut -d' ' -f1)" = "${NODEDC_MODULE_SHA256}" \
&& chmod 0444 /opt/nodedc/adapter/*.py \
&& cd /opt/nodedc/adapter \
&& python3 -B -m py_compile portable_lab_v1_component_adapter.py \
run_portable_lab_v1_eomt_component.py run_portable_camera_source_component.py \
&& rm -rf /opt/nodedc/adapter/__pycache__
LABEL org.opencontainers.image.revision="${NODEDC_REVISION}" \
com.nodedc.product="mission-core" \
com.nodedc.stack="observatory" \
com.nodedc.role="ai-module" \
com.nodedc.module-id="camera-source" \
com.nodedc.managed-by="mission-core-worker"
ENTRYPOINT ["python3"]
CMD ["/opt/nodedc/adapter/run_portable_camera_source_component.py"]
@@ -0,0 +1,26 @@
FROM ndc/mission-core-installed-lab-v1-ddrnet-step:439127908dba
ARG NODEDC_SHARED_SHA256
ARG NODEDC_MODULE_SHA256
ARG NODEDC_REVISION
COPY portable_lab_v1_component_adapter.py /opt/nodedc/adapter/portable_lab_v1_component_adapter.py
COPY run_portable_lab_v1_ddrnet_component.py /opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py
RUN test "$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py | cut -d' ' -f1)" = "${NODEDC_SHARED_SHA256}" \
&& test "$(sha256sum /opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py | cut -d' ' -f1)" = "${NODEDC_MODULE_SHA256}" \
&& chmod 0444 /opt/nodedc/adapter/*.py \
&& cd /opt/nodedc/adapter \
&& conda run --no-capture-output --name goose python -B -m py_compile \
portable_lab_v1_component_adapter.py run_portable_lab_v1_ddrnet_component.py \
&& rm -rf /opt/nodedc/adapter/__pycache__
LABEL org.opencontainers.image.revision="${NODEDC_REVISION}" \
com.nodedc.product="mission-core" \
com.nodedc.stack="observatory" \
com.nodedc.role="ai-module" \
com.nodedc.module-id="ddrnet" \
com.nodedc.managed-by="mission-core-worker"
ENTRYPOINT ["conda","run","--no-capture-output","--name","goose","python"]
CMD ["/opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py"]
@@ -0,0 +1,26 @@
FROM ndc/mission-core-installed-lab-v1-eomt-step:ee0efdd9af72
ARG NODEDC_SHARED_SHA256
ARG NODEDC_MODULE_SHA256
ARG NODEDC_REVISION
COPY portable_lab_v1_component_adapter.py /opt/nodedc/adapter/portable_lab_v1_component_adapter.py
COPY run_portable_lab_v1_eomt_component.py /opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py
RUN test "$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py | cut -d' ' -f1)" = "${NODEDC_SHARED_SHA256}" \
&& test "$(sha256sum /opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py | cut -d' ' -f1)" = "${NODEDC_MODULE_SHA256}" \
&& chmod 0444 /opt/nodedc/adapter/*.py \
&& cd /opt/nodedc/adapter \
&& python3 -B -m py_compile portable_lab_v1_component_adapter.py \
run_portable_lab_v1_eomt_component.py \
&& rm -rf /opt/nodedc/adapter/__pycache__
LABEL org.opencontainers.image.revision="${NODEDC_REVISION}" \
com.nodedc.product="mission-core" \
com.nodedc.stack="observatory" \
com.nodedc.role="ai-module" \
com.nodedc.module-id="eomt" \
com.nodedc.managed-by="mission-core-worker"
ENTRYPOINT ["python3"]
CMD ["/opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py"]
@@ -0,0 +1,20 @@
FROM ndc-k1-perception-ddrnet39-rfdetr-tgs:stage1-joint-20260901
ARG NODEDC_MODULE_SHA256
ARG NODEDC_REVISION
COPY run_ai_module_object_distance.py /opt/nodedc/adapter/run_ai_module_object_distance.py
RUN test "$(sha256sum /opt/nodedc/adapter/run_ai_module_object_distance.py | cut -d' ' -f1)" = "${NODEDC_MODULE_SHA256}" \
&& chmod 0444 /opt/nodedc/adapter/run_ai_module_object_distance.py \
&& python3 -B -m py_compile /opt/nodedc/adapter/run_ai_module_object_distance.py \
&& rm -rf /opt/nodedc/adapter/__pycache__
LABEL org.opencontainers.image.revision="${NODEDC_REVISION}" \
com.nodedc.product="mission-core" \
com.nodedc.stack="observatory" \
com.nodedc.role="ai-module" \
com.nodedc.module-id="object-distance" \
com.nodedc.managed-by="mission-core-worker"
ENTRYPOINT ["python3", "-B", "/opt/nodedc/adapter/run_ai_module_object_distance.py"]
@@ -0,0 +1,20 @@
FROM ndc-k1-perception-ddrnet39-rfdetr-tgs:stage1-joint-20260901
ARG NODEDC_MODULE_SHA256
ARG NODEDC_REVISION
COPY run_ai_module_rf_detr.py /opt/nodedc/adapter/run_ai_module_rf_detr.py
RUN test "$(sha256sum /opt/nodedc/adapter/run_ai_module_rf_detr.py | cut -d' ' -f1)" = "${NODEDC_MODULE_SHA256}" \
&& chmod 0444 /opt/nodedc/adapter/run_ai_module_rf_detr.py \
&& python3 -B -m py_compile /opt/nodedc/adapter/run_ai_module_rf_detr.py \
&& rm -rf /opt/nodedc/adapter/__pycache__
LABEL org.opencontainers.image.revision="${NODEDC_REVISION}" \
com.nodedc.product="mission-core" \
com.nodedc.stack="observatory" \
com.nodedc.role="ai-module" \
com.nodedc.module-id="rf-detr" \
com.nodedc.managed-by="mission-core-worker"
ENTRYPOINT ["python3", "-B", "/opt/nodedc/adapter/run_ai_module_rf_detr.py"]
@@ -15,7 +15,7 @@ RUN case "${NODEDC_SOURCE_TREE_SHA256}" in *[!0-9a-f]*|'') exit 64 ;; esac \
&& test "${#NODEDC_SOURCE_TREE_SHA256}" -eq 64 \
&& find /opt/nodedc/mission-core/src/k1link -type d -exec chmod 0555 {} + \
&& find /opt/nodedc/mission-core/src/k1link -type f -exec chmod 0444 {} + \
&& python3 -B -c "import k1link.observatory.installed_lab_worker_container_main as entrypoint; import k1link.observatory.installed_lab_worker_service as worker; import k1link.observatory.lab_v1_installed_package_steps as steps; assert callable(entrypoint.main); assert callable(worker.main); assert callable(steps.main)"
&& python3 -B -c "import k1link.observatory.installed_lab_worker_container_main as entrypoint; import k1link.observatory.installed_lab_worker_service as worker; import k1link.observatory.lab_v1_installed_package_steps as legacy_steps; import k1link.observatory.modular_installed_package_steps as modular_steps; assert callable(entrypoint.main); assert callable(worker.main); assert callable(legacy_steps.main); assert callable(modular_steps.main)"
LABEL com.nodedc.product="mission-core" \
com.nodedc.stack="observatory" \
@@ -0,0 +1,281 @@
"""Idle-only two-file control-agent update; sealed compute packages are unchanged.
Offline child images, exact parent/source/create fences, immutable install
receipts, and stopped predecessors retained for explicit rollback. No model
execution, new volume, new resource limit, or queue mutation occurs here.
"""
from __future__ import annotations
import argparse
import copy
import io
import json
import tarfile
import time
from datetime import UTC, datetime
from pathlib import Path
from urllib.parse import urlencode
from migrate_claim_transport_v3 import READINESS, Engine, canonical, require_idle, save, sha
SOURCE_ROOT = "/opt/nodedc/installed-lab/src/k1link/observatory"
BEFORE = {
"worker_agent.py": "91a65fcde45fa65b0894e6927618f35b4d8aa369653d00163df7fd840e6ea5b2",
"worker_http_transport.py": "dfe652d9464d97cba37c4136be8d0fd865f3e93fe75380e471da882b56c517f1",
}
TARGETS = {
"ndc-observatory-m49-worker-agent": (
"d252326dba36a1d4e4194f2862078a00090e93439c03f4d6cef97bf8ef43a607"
),
"ndc-observatory-installed-lab-worker-agent": (
"b9131e995b14a8e42fbf0e5bf0e017c93ce1af33c0910927ae5e4514d900e683"
),
}
LABEL = "com.nodedc.recorded-heartbeat-recovery.plan-sha256"
def probe() -> str:
return f"""import hashlib,json,pathlib
from k1link.observatory import worker_agent
root=pathlib.Path(worker_agent.__file__).resolve().parent
assert str(root)=={SOURCE_ROOT!r}
print(json.dumps({{n:hashlib.sha256((root/n).read_bytes()).hexdigest() for n in {list(BEFORE)!r}}}))
"""
def pack(repository: Path, output: Path) -> None:
output.mkdir(parents=False, exist_ok=False)
files = {}
for name in BEFORE:
payload = (repository / "src/k1link/observatory" / name).read_bytes()
compile(payload, name, "exec")
(output / name).write_bytes(payload)
files[name] = sha(payload)
save(output / "payload.json", {"schema_version": 1, "files": files})
def payload_files(root: Path) -> dict[str, bytes]:
manifest = json.loads((root / "payload.json").read_bytes())
if set(manifest) != {"schema_version", "files"} or manifest["schema_version"] != 1:
raise ValueError("invalid heartbeat payload manifest")
if set(manifest["files"]) != set(BEFORE):
raise ValueError("heartbeat file set changed")
result = {}
for name in BEFORE:
path = root / name
if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size < 256_000:
raise ValueError("unsafe heartbeat payload")
value = path.read_bytes()
if sha(value) != manifest["files"][name]:
raise ValueError("heartbeat payload changed")
compile(value, name, "exec")
result[name] = value
return result
def create_hash(row: dict) -> str:
return sha(canonical({"Config": row["Config"], "HostConfig": row["HostConfig"]}))
def validate_target(name: str, row: dict) -> None:
config, host = row["Config"], row["HostConfig"]
if row["Name"] != "/" + name or row["Image"] != "sha256:" + TARGETS[name]:
raise ValueError("control-agent identity changed")
if not row["State"]["Running"] or not host["ReadonlyRootfs"]:
raise ValueError("expected durable read-only agent is not running")
if host["NetworkMode"] != "bridge" or host.get("DeviceRequests") or host.get("Privileged"):
raise ValueError("control-agent GPU/network boundary changed")
if config["Labels"].get("com.nodedc.authority") != "observation-only":
raise ValueError("control-agent authority changed")
for item in config["Env"]:
key = item.split("=", 1)[0].upper()
if any(word in key for word in ("TOKEN", "PASSWORD", "SECRET")) and not key.endswith(
"_FILE"
):
raise ValueError("inline credential is forbidden")
def plan(engine: Engine, root: Path) -> dict:
files, targets = payload_files(root), []
for name, parent in TARGETS.items():
row = engine.inspect(name)
validate_target(name, row)
if engine.execute_json(name, probe()) != BEFORE:
raise ValueError("imported source differs from reviewed baseline")
require_idle(engine.execute_json(name, READINESS))
targets.append(
{"name": name, "id": row["Id"], "parent": parent, "create_sha256": create_hash(row)}
)
return {
"schema_version": "missioncore.recorded-heartbeat-recovery-install/v1",
"targets": targets,
"files": {name: sha(value) for name, value in files.items()},
"installer_sha256": sha(Path(__file__).read_bytes()),
"helper_sha256": sha(
Path(__file__).with_name("migrate_claim_transport_v3.py").read_bytes()
),
"compute_packages_changed": False,
}
def fence(engine: Engine, target: dict) -> dict:
row = engine.inspect(target["name"])
validate_target(target["name"], row)
if row["Id"] != target["id"] or create_hash(row) != target["create_sha256"]:
raise ValueError("control agent changed since plan")
require_idle(engine.execute_json(target["name"], READINESS))
return row
def build(engine: Engine, target: dict, files: dict[str, bytes], plan_sha: str) -> str:
created = engine.request(
"POST",
"/containers/create",
{
"Image": "sha256:" + target["parent"],
"Entrypoint": ["/bin/true"],
"Cmd": [],
"HostConfig": {
"NetworkMode": "none",
"CapDrop": ["ALL"],
"PidsLimit": 32,
"SecurityOpt": ["no-new-privileges"],
},
},
)["Id"]
try:
engine.request("POST", f"/containers/{created}/start")
if engine.request("POST", f"/containers/{created}/wait")["StatusCode"] != 0:
raise ValueError("offline layer initialization failed")
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w") as archive:
for name, payload in files.items():
item = tarfile.TarInfo(name)
item.size, item.mode, item.mtime = len(payload), 0o444, int(time.time())
archive.addfile(item, io.BytesIO(payload))
engine.request(
"PUT",
f"/containers/{created}/archive?" + urlencode({"path": SOURCE_ROOT}),
buffer.getvalue(),
)
changes = engine.request("GET", f"/containers/{created}/changes")
allowed = {str(Path(SOURCE_ROOT) / name) for name in files}
parents = {str(parent) for name in allowed for parent in Path(name).parents}
if not changes or any(
row["Path"] not in allowed | parents or row["Kind"] not in (0, 1) for row in changes
):
raise ValueError("unrelated changes in offline heartbeat layer")
if not allowed.issubset({row["Path"] for row in changes}):
raise ValueError("heartbeat layer omitted a file")
parent = engine.request("GET", f"/images/sha256:{target['parent']}/json")
config = copy.deepcopy(parent["Config"])
config.setdefault("Labels", {})[LABEL] = plan_sha
image = engine.request(
"POST",
"/commit?"
+ urlencode(
{
"container": created,
"repo": target["name"] + "-heartbeat-recovery",
"tag": "v1",
}
),
config,
)["Id"]
after = engine.request("GET", f"/images/{image}/json")
if after["RootFS"]["Layers"][:-1] != parent["RootFS"]["Layers"]:
raise ValueError("parent image layers changed")
return image
finally:
engine.request("DELETE", f"/containers/{created}")
def apply(engine: Engine, root: Path, expected: str, evidence: Path) -> dict:
started, mono = datetime.now(UTC).isoformat(), time.monotonic_ns()
proposal = plan(engine, root)
if sha(canonical(proposal)) != expected:
raise ValueError("heartbeat install plan changed")
evidence.mkdir(parents=False, exist_ok=False)
save(evidence / "plan.json", proposal)
files, results = payload_files(root), []
for target in proposal["targets"]:
name = target["name"]
fence(engine, target)
image = build(engine, target, files, expected)
before = fence(engine, target)
body = copy.deepcopy(before["Config"])
body["Image"] = image
body["Labels"][LABEL] = expected
body["HostConfig"] = copy.deepcopy(before["HostConfig"])
backup = name + "-pre-heartbeat-" + before["Id"][:12]
save(
evidence / (name + "-declaration.json"),
{
"name": name,
"create_body": body,
"rollback_name": backup,
"rollback_container_id": before["Id"],
"parent": target["parent"],
},
)
engine.request("POST", f"/containers/{before['Id']}/stop?t=15")
engine.request(
"POST", f"/containers/{before['Id']}/update", {"RestartPolicy": {"Name": "no"}}
)
engine.request("POST", f"/containers/{before['Id']}/rename?" + urlencode({"name": backup}))
created = engine.request("POST", "/containers/create?" + urlencode({"name": name}), body)[
"Id"
]
engine.request("POST", f"/containers/{created}/start")
time.sleep(3)
after = engine.inspect(name)
if not after["State"]["Running"] or after["RestartCount"] != 0:
raise ValueError("replacement requires reconciliation; predecessor retained")
if engine.execute_json(name, probe()) != proposal["files"]:
raise ValueError("replacement imported another payload")
results.append(
{
"name": name,
"id": created,
"image": image,
"rollback": backup,
"readiness": engine.execute_json(name, READINESS),
}
)
save(evidence / (name + "-acceptance.json"), results[-1])
receipt = {
"plan_sha256": expected,
"agents": results,
"compute_packages_changed": False,
"started_at_utc": started,
"finished_at_utc": datetime.now(UTC).isoformat(),
"started_monotonic_ns": mono,
"finished_monotonic_ns": time.monotonic_ns(),
}
save(evidence / "receipt.json", receipt)
return receipt
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repository", type=Path)
parser.add_argument("--pack", type=Path)
parser.add_argument("--payload", type=Path)
parser.add_argument("--apply-plan-sha256")
parser.add_argument("--evidence", type=Path)
args = parser.parse_args()
if args.pack:
pack(args.repository, args.pack)
return
engine = Engine()
if args.apply_plan_sha256:
document = apply(engine, args.payload, args.apply_plan_sha256, args.evidence)
else:
document = plan(engine, args.payload)
document = {"plan": document, "plan_sha256": sha(canonical(document))}
print(json.dumps(document, sort_keys=True))
if __name__ == "__main__":
main()
@@ -0,0 +1,340 @@
"""Exact offline control-agent update: shared source cache and LiDAR reuse.
Plan is read-only, apply requires its hash and an idle queue. Models, package
definitions, legacy LiDAR producer and existing work/results stay unchanged.
Stopped predecessors and full create declarations are retained for rollback.
"""
from __future__ import annotations
import argparse
import copy
import io
import json
import tarfile
import time
from datetime import UTC, datetime
from pathlib import Path
from urllib.parse import urlencode
from migrate_claim_transport_v3 import READINESS, Engine, canonical, require_idle, save, sha
SOURCE_ROOT = "/opt/nodedc/installed-lab/src/k1link"
PRODUCER = "compute/lidar_replay.py"
PRODUCER_SHA = "543a1d63889ad513e6307603cf477f937645c1e4df9a63a169424afd9d2471b8"
BEFORE = {
"compute/lidar_preparation.py": None,
"observatory/m49_portable_source.py": (
"aff9baf5a11732c3f5d4bf53a5a3306df8ac020efbb0dc235202f7193b43c043"
),
"observatory/worker_http_transport.py": (
"363daa574139ee062f0d4141c6ce8ca9c3f25883e1fee6660dd771e15ec71088"
),
"observatory/worker_service.py": (
"a742d1de10e9c76e531f5be78935d6c34c8195d4ecc8c24cf25b987ec162ba1a"
),
"observatory/worker_source_cache.py": None,
}
TARGETS = {
"ndc-observatory-m49-worker-agent": (
"927c3c4f5b00ae6c084d1f5a8bc77f7b262cfc5e83cf4be1c48f615c12c06e80"
),
"ndc-observatory-installed-lab-worker-agent": (
"052af3ccd10e11b162c163943f5427af2dc95b09b93481dd85e954e494ba1107"
),
}
VOLUME = "ndc-observatory-source-cas-v1"
CACHE_PATH = "/source-cache"
CACHE_ENV = "MISSIONCORE_OBSERVATORY_WORKER_SOURCE_CACHE_ROOT"
LABELS = {
"com.nodedc.product": "mission-core",
"com.nodedc.stack": "observatory",
"com.nodedc.role": "source-cache",
"com.nodedc.managed-by": "recorded-source-reuse-v1",
}
def probe() -> str:
names = [PRODUCER, *BEFORE]
return f"""import hashlib,json,pathlib
from k1link.observatory import worker_http_transport
root=pathlib.Path(worker_http_transport.__file__).resolve().parents[1]
assert str(root)=={SOURCE_ROOT!r}
print(json.dumps({{n:hashlib.sha256((root/n).read_bytes()).hexdigest()
if (root/n).is_file() else None for n in {names!r}}}))
"""
def pack(repository: Path, output: Path) -> None:
if sha((repository / "src/k1link" / PRODUCER).read_bytes()) != PRODUCER_SHA:
raise ValueError("legacy LiDAR producer changed")
output.mkdir(parents=False, exist_ok=False)
files = {}
for name in BEFORE:
payload = (repository / "src/k1link" / name).read_bytes()
compile(payload, name, "exec")
target = output / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(payload)
files[name] = sha(payload)
save(output / "payload.json", {"schema_version": 1, "files": files})
def payload_files(root: Path) -> dict[str, bytes]:
manifest = json.loads((root / "payload.json").read_bytes())
if set(manifest) != {"schema_version", "files"} or manifest["schema_version"] != 1:
raise ValueError("invalid source-reuse payload manifest")
if set(manifest["files"]) != set(BEFORE):
raise ValueError("source-reuse file set changed")
result = {}
for name in BEFORE:
path = root / name
if path.is_symlink() or not path.is_file() or path.stat().st_size > 256_000:
raise ValueError("unsafe source-reuse payload")
value = path.read_bytes()
if sha(value) != manifest["files"][name]:
raise ValueError("source-reuse payload changed")
compile(value, name, "exec")
result[name] = value
return result
def create_hash(row: dict) -> str:
return sha(canonical({"Config": row["Config"], "HostConfig": row["HostConfig"]}))
def validate_target(name: str, row: dict) -> None:
config, host = row["Config"], row["HostConfig"]
if row["Name"] != "/" + name or row["Image"] != "sha256:" + TARGETS[name]:
raise ValueError("control-agent identity changed")
if not row["State"]["Running"] or not host["ReadonlyRootfs"]:
raise ValueError("control agent must be running/read-only")
if host["NetworkMode"] != "bridge" or host.get("DeviceRequests") or host.get("Privileged"):
raise ValueError("control-agent network/GPU boundary changed")
if config["Labels"].get("com.nodedc.authority") != "observation-only":
raise ValueError("control-agent authority changed")
if any(mount["Destination"] == CACHE_PATH for mount in row["Mounts"]):
raise ValueError("source-cache mount is already occupied")
for entry in config["Env"]:
key = entry.split("=", 1)[0]
if key == CACHE_ENV:
raise ValueError("source-cache configuration already exists")
if any(
word in key.upper() for word in ("TOKEN", "PASSWORD", "SECRET")
) and not key.endswith("_FILE"):
raise ValueError("inline secret is forbidden in saved declarations")
def volume_state(engine: Engine) -> dict:
response = engine.request(
"GET", "/volumes?" + urlencode({"filters": json.dumps({"name": [VOLUME]})})
)
matches = [row for row in response.get("Volumes", []) or [] if row["Name"] == VOLUME]
if not matches:
return {"exists": False}
row = matches[0]
if row["Driver"] != "local" or row.get("Labels") != LABELS:
raise ValueError("existing source-cache volume has another owner")
return {"exists": True, "name": VOLUME, "driver": "local", "labels": LABELS}
def plan(engine: Engine, root: Path) -> dict:
files = payload_files(root)
targets = []
for name, parent in TARGETS.items():
row = engine.inspect(name)
validate_target(name, row)
if engine.execute_json(name, probe()) != {PRODUCER: PRODUCER_SHA, **BEFORE}:
raise ValueError("imported source differs from reviewed baseline")
require_idle(engine.execute_json(name, READINESS))
targets.append(
{"name": name, "id": row["Id"], "parent": parent, "create_sha256": create_hash(row)}
)
return {
"schema_version": "missioncore.recorded-source-reuse-install-plan/v1",
"targets": targets,
"files": {name: sha(value) for name, value in files.items()},
"producer_sha256": PRODUCER_SHA,
"shared_cache": {"name": VOLUME, "target": CACHE_PATH, "before": volume_state(engine)},
"installer_sha256": sha(Path(__file__).read_bytes()),
"engine_helper_sha256": sha(
Path(__file__).with_name("migrate_claim_transport_v3.py").read_bytes()
),
"compute_packages_changed": False,
}
def fence(engine: Engine, target: dict) -> dict:
row = engine.inspect(target["name"])
validate_target(target["name"], row)
if row["Id"] != target["id"] or create_hash(row) != target["create_sha256"]:
raise ValueError("control agent changed since plan")
require_idle(engine.execute_json(target["name"], READINESS))
return row
def build(engine: Engine, target: dict, files: dict[str, bytes], plan_sha: str) -> str:
created = engine.request(
"POST",
"/containers/create",
{
"Image": "sha256:" + target["parent"],
"Entrypoint": ["/bin/true"],
"Cmd": [],
"HostConfig": {
"NetworkMode": "none",
"CapDrop": ["ALL"],
"PidsLimit": 32,
"SecurityOpt": ["no-new-privileges"],
},
},
)["Id"]
try:
engine.request("POST", f"/containers/{created}/start")
if engine.request("POST", f"/containers/{created}/wait")["StatusCode"] != 0:
raise ValueError("offline layer initialization failed")
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w") as archive:
for name, payload in files.items():
item = tarfile.TarInfo(name)
item.size, item.mode, item.mtime = len(payload), 0o444, int(time.time())
archive.addfile(item, io.BytesIO(payload))
engine.request(
"PUT",
f"/containers/{created}/archive?" + urlencode({"path": SOURCE_ROOT}),
buffer.getvalue(),
)
changes = engine.request("GET", f"/containers/{created}/changes")
allowed = {str(Path(SOURCE_ROOT) / name) for name in files}
parents = {str(parent) for name in allowed for parent in Path(name).parents}
if not changes or any(
row["Path"] not in allowed | parents or row["Kind"] not in (0, 1) for row in changes
):
raise ValueError("unrelated changes in offline source-reuse layer")
if not allowed.issubset({row["Path"] for row in changes}):
raise ValueError("source-reuse layer omitted a file")
parent = engine.request("GET", f"/images/sha256:{target['parent']}/json")
config = copy.deepcopy(parent["Config"])
config.setdefault("Labels", {})["com.nodedc.source-reuse.plan-sha256"] = plan_sha
image = engine.request(
"POST",
"/commit?"
+ urlencode(
{
"container": created,
"repo": target["name"] + "-source-reuse",
"tag": "v1",
}
),
config,
)["Id"]
after = engine.request("GET", f"/images/{image}/json")
if after["RootFS"]["Layers"][:-1] != parent["RootFS"]["Layers"]:
raise ValueError("parent image layers changed")
return image
finally:
engine.request("DELETE", f"/containers/{created}")
def apply(engine: Engine, root: Path, expected: str, evidence: Path) -> dict:
started_at, started_mono = datetime.now(UTC).isoformat(), time.monotonic_ns()
proposal = plan(engine, root)
if sha(canonical(proposal)) != expected:
raise ValueError("source-reuse install plan changed")
evidence.mkdir(parents=False, exist_ok=False)
save(evidence / "plan.json", proposal)
files = payload_files(root)
engine.request("POST", "/volumes/create", {"Name": VOLUME, "Driver": "local", "Labels": LABELS})
volume_state(engine)
results = []
for target in proposal["targets"]:
name = target["name"]
fence(engine, target)
image = build(engine, target, files, expected)
before = fence(engine, target)
body = copy.deepcopy(before["Config"])
body["Image"] = image
body["Env"].append(CACHE_ENV + "=" + CACHE_PATH)
body["Labels"]["com.nodedc.source-reuse.plan-sha256"] = expected
body["HostConfig"] = copy.deepcopy(before["HostConfig"])
body["HostConfig"].setdefault("Mounts", []).append(
{
"Type": "volume",
"Source": VOLUME,
"Target": CACHE_PATH,
"ReadOnly": False,
}
)
backup = name + "-pre-source-reuse-" + before["Id"][:12]
save(
evidence / (name + "-declaration.json"),
{
"name": name,
"create_body": body,
"rollback_name": backup,
"rollback_container_id": before["Id"],
"parent": target["parent"],
},
)
engine.request("POST", f"/containers/{before['Id']}/stop?t=15")
engine.request(
"POST", f"/containers/{before['Id']}/update", {"RestartPolicy": {"Name": "no"}}
)
engine.request("POST", f"/containers/{before['Id']}/rename?" + urlencode({"name": backup}))
created = engine.request("POST", "/containers/create?" + urlencode({"name": name}), body)[
"Id"
]
engine.request("POST", f"/containers/{created}/start")
# Never auto-delete a replacement: it may already own operator work.
time.sleep(3)
after = engine.inspect(name)
if not after["State"]["Running"] or after["RestartCount"] != 0:
raise ValueError("replacement requires reconciliation; predecessor retained")
if engine.execute_json(name, probe()) != {PRODUCER: PRODUCER_SHA, **proposal["files"]}:
raise ValueError("replacement imported another payload")
results.append(
{
"name": name,
"id": created,
"image": image,
"rollback": backup,
"readiness": engine.execute_json(name, READINESS),
}
)
save(evidence / (name + "-acceptance.json"), results[-1])
receipt = {
"plan_sha256": expected,
"agents": results,
"shared_cache": VOLUME,
"compute_packages_changed": False,
"started_at_utc": started_at,
"finished_at_utc": datetime.now(UTC).isoformat(),
"started_monotonic_ns": started_mono,
"finished_monotonic_ns": time.monotonic_ns(),
}
save(evidence / "receipt.json", receipt)
return receipt
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repository", type=Path)
parser.add_argument("--pack", type=Path)
parser.add_argument("--payload", type=Path)
parser.add_argument("--apply-plan-sha256")
parser.add_argument("--evidence", type=Path)
args = parser.parse_args()
if args.pack:
pack(args.repository, args.pack)
return
engine = Engine()
if args.apply_plan_sha256:
result = apply(engine, args.payload, args.apply_plan_sha256, args.evidence)
else:
proposal = plan(engine, args.payload)
result = {"plan": proposal, "plan_sha256": sha(canonical(proposal))}
print(json.dumps(result, sort_keys=True))
if __name__ == "__main__":
main()
@@ -71,7 +71,7 @@ _SAFE_COMPONENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
_OBSERVATORY_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$")
_CAMERA_JOB_ID = re.compile(r"^recorded-camera-[a-f0-9]{24}$")
Component = Literal["eomt", "ddrnet"]
Component = Literal["camera-source", "eomt", "ddrnet"]
AssetKind = Literal["file", "tree"]
AssetVerification = Literal["sha256", "identity-sha256"]
CommandRunner = Callable[[Sequence[str], Optional[Mapping[str, str]]], None] # noqa: UP045
@@ -168,7 +168,7 @@ class RuntimeLayout:
component: Component,
expectations: Sequence[AssetExpectation],
) -> RuntimeLayout:
if component not in ("eomt", "ddrnet"):
if component not in ("camera-source", "eomt", "ddrnet"):
raise ComponentAdapterError("installed package component is invalid")
prepared = Path(PACKAGE_STEP_INPUT_ROOT) / "prepare"
return cls(
@@ -176,7 +176,7 @@ class RuntimeLayout:
camera_job_root=prepared / "camera-job",
output_root=Path(PACKAGE_OUTPUT_ROOT),
effective_ddrnet_config=prepared / "effective-ddrnet-config.json",
eomt_result_root=Path(PACKAGE_STEP_INPUT_ROOT) / "eomt",
eomt_result_root=Path(PACKAGE_STEP_INPUT_ROOT) / "camera-source",
asset_paths={item.asset_id: Path(item.path) for item in expectations},
)
@@ -289,7 +289,7 @@ def load_component_request(
source = _source_binding(document["source"])
paths = _paths(document["paths"], component)
effective_value = document["effective_ddrnet_config_sha256"]
if component == "eomt":
if component != "ddrnet":
if effective_value is not None:
raise ComponentAdapterError("EoMT request contains a DDRNet config")
effective_sha256: str | None = None
@@ -440,6 +440,61 @@ def validate_tree_asset(
return _real_directory(layout.asset_paths[asset_id], f"asset {asset_id}")
def validate_prepared_camera_root(
root: Path,
request: ComponentRequest,
) -> dict[str, Path]:
"""Verify the model-neutral camera-source output before any AI consumes it."""
resolved = _real_directory(root, "prepared camera root")
expected_children = {
"camera-source.json",
"decode-repair.json",
"source-frames",
"source-frames.json",
"timeline.jsonl",
}
if {path.name for path in resolved.iterdir()} != expected_children:
raise ComponentAdapterError("prepared camera artifact set changed")
receipt = load_canonical_json(
resolved / "camera-source.json",
label="prepared camera receipt",
maximum=1024 * 1024,
confinement_root=resolved,
)
source = request.source
if receipt.get("schema_version") != "missioncore.observatory-prepared-camera/v1" or receipt.get(
"source"
) != {
"camera_job_id": source.camera_job_id,
"input_sha256": source.camera_input_sha256,
"frame_count": source.frame_count,
}:
raise ComponentAdapterError("prepared camera identity changed")
rows = receipt.get("artifacts")
expected = {
"decode-repair": "decode-repair.json",
"source-frames": "source-frames.json",
"timeline": "timeline.jsonl",
}
if not isinstance(rows, list) or len(rows) != len(expected):
raise ComponentAdapterError("prepared camera receipt changed")
result: dict[str, Path] = {}
for row in rows:
if not isinstance(row, dict) or set(row) != {"role", "path", "byte_length", "sha256"}:
raise ComponentAdapterError("prepared camera artifact changed")
role = row.get("role")
name = expected.get(role) if isinstance(role, str) else None
if name != row.get("path"):
raise ComponentAdapterError("prepared camera artifact role changed")
path = _real_file(resolved / name, "prepared camera artifact")
if path.parent != resolved or (
row.get("byte_length") != path.stat().st_size or row.get("sha256") != sha256_file(path)
):
raise ComponentAdapterError("prepared camera artifact identity changed")
result[role] = path
return result
def validate_tree_receipt(
request: ComponentRequest,
root: Path,
@@ -833,12 +888,12 @@ def _paths(value: object, component: Component) -> dict[str, str | None]:
f"{prepared}/effective-ddrnet-config.json" if component == "ddrnet" else None
),
"eomt_result_root": (
f"{PACKAGE_STEP_INPUT_ROOT}/eomt" if component == "ddrnet" else None
f"{PACKAGE_STEP_INPUT_ROOT}/camera-source" if component == "ddrnet" else None
),
"decoded_frames_root": (
f"{PACKAGE_OUTPUT_ROOT}/source-frames"
if component == "eomt"
else f"{PACKAGE_STEP_INPUT_ROOT}/eomt/source-frames"
if component == "camera-source"
else f"{PACKAGE_STEP_INPUT_ROOT}/camera-source/source-frames"
),
}
_exact_keys(document, set(legacy), "component paths")
@@ -0,0 +1,63 @@
"""Small synthetic shared-cache proof; never claims work or loads a model."""
from __future__ import annotations
import argparse
import hashlib
import json
import tempfile
import time
from datetime import UTC, datetime
from pathlib import Path
from k1link.observatory.worker_service import ObservatoryWorkerServiceConfiguration
from k1link.observatory.worker_source_cache import WorkerSourceCache
PAYLOAD = b"missioncore-source-reuse-proof-AsldTM-v1\n" * 4096
SHA256 = hashlib.sha256(PAYLOAD).hexdigest()
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=("retain", "restore", "cleanup"))
args = parser.parse_args()
configuration = ObservatoryWorkerServiceConfiguration.from_environment()
assert configuration.source_cache_root == Path("/source-cache")
started, mono = datetime.now(UTC).isoformat(), time.monotonic_ns()
cache = WorkerSourceCache(configuration.source_cache_root)
with tempfile.TemporaryDirectory(
prefix=".source-reuse-proof-", dir=configuration.work_root
) as root:
source = Path(root) / "synthetic.bin"
if args.mode == "retain":
source.write_bytes(PAYLOAD)
assert cache.retain(source, sha256=SHA256, byte_length=len(PAYLOAD))
else:
assert cache.restore(source, sha256=SHA256, byte_length=len(PAYLOAD))
assert source.read_bytes() == PAYLOAD
if args.mode == "cleanup":
cached = cache.root / SHA256
assert cached.read_bytes() == PAYLOAD
cached.unlink() # Only this probe's verified synthetic cache object.
print(
json.dumps(
{
"schema_version": "missioncore.worker-source-cache-proof/v1",
"mode": args.mode,
"source_sha256": SHA256,
"byte_length": len(PAYLOAD),
"exact": True,
"model_jobs": 0,
"temporary_work_removed": True,
"synthetic_cache_removed": args.mode == "cleanup",
"started_at_utc": started,
"finished_at_utc": datetime.now(UTC).isoformat(),
"started_monotonic_ns": mono,
"finished_monotonic_ns": time.monotonic_ns(),
}
)
)
if __name__ == "__main__":
main()
@@ -0,0 +1,546 @@
#!/usr/bin/env python3
"""Associate RF-DETR boxes with current K1 LiDAR and publish metric ranges."""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import re
import sys
import time
import zipfile
from dataclasses import dataclass
from pathlib import Path
from typing import cast
import numpy as np
from k1link.compute.lidar_local_surface_shadow import (
K1LocalSurfaceShadowEstimator,
K1LocalSurfaceShadowInput,
)
from k1link.perception.contracts import (
ClockBasis,
ModalityOutcome,
ModalityStatus,
ObjectProposal2D,
SourceEnvelope,
TimestampBundle,
)
from k1link.perception.geometry import (
GEOMETRY_PROVIDER_ID,
GeometryFrame,
GeometryProfile,
Ravnoves00GeometryAssociationProvider,
)
from k1link.perception.geometry_math import GeometryAssociationProfile, Kb4ProjectionProfile
from k1link.perception.providers import SourcePacket
SCHEMA = "missioncore.observatory-ai-module-object-distance-result/v1"
ROW_SCHEMA = "missioncore.observatory-ai-module-object-distance-frame/v1"
RF_ROW_SCHEMA = "missioncore.observatory-ai-module-rf-detr-frame/v1"
AUTHORITY = {
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
}
_PACKAGE_SOURCE = Path("/missioncore/input/steps/prepare/source-input.json")
_PACKAGE_DETECTIONS = Path("/missioncore/input/steps/rf-detr/detections.jsonl")
_PACKAGE_LIDAR_PACK = Path("/missioncore/input/steps/prepare/lidar-pack")
_PACKAGE_BINDING_INDEX = Path("/missioncore/input/steps/prepare/m49-source/sequence-index.ndjson")
_PACKAGE_CALIBRATION = Path("/opt/nodedc/assets/k1-camera-lidar-calibration")
_PACKAGE_OUTPUT = Path("/missioncore/output")
_MAX_CALIBRATION_PACK_BYTES = 128 * 1024 * 1024
_LIDAR_PACK_ID = re.compile(r"^lidar-replay-pack-([a-f0-9]{64})$")
ASSOCIATION = GeometryAssociationProfile(
bbox_inset_fraction=0.03,
depth_cluster_minimum_gap_m=0.45,
depth_cluster_gap_fraction=0.08,
spatial_cluster_radius_m=0.6,
semantic_minimum_occupied_points=2,
semantic_minimum_occupied_voxels=1,
semantic_voxel_size_m=0.35,
conflict_minimum_classified_points=6,
conflict_surface_fraction=0.8,
geometry_local_radius_m=10.0,
geometry_voxel_size_m=0.45,
geometry_minimum_cluster_points=4,
geometry_minimum_cluster_voxels=1,
maximum_geometry_clusters_per_frame=64,
)
class ObjectDistanceModuleError(RuntimeError):
pass
class _CurrentStore:
def __init__(self, profile: GeometryProfile) -> None:
self.profile = profile
self.current: GeometryFrame | None = None
def frame(self, _packet: SourcePacket) -> GeometryFrame | None:
return self.current
@dataclass(frozen=True)
class _PointFrame:
received_monotonic_ns: int
xyz_map: np.ndarray
@dataclass(frozen=True)
class _PoseFrame:
received_monotonic_ns: int
position_map: tuple[float, float, float]
orientation_map_from_lidar: tuple[float, float, float, float]
class _LidarPack:
"""Narrow reader for the already sealed replay-pack arrays."""
def __init__(self, root: Path) -> None:
candidate = root.expanduser().absolute()
if candidate.is_symlink():
raise ObjectDistanceModuleError("LiDAR replay pack identity changed")
self.root = candidate.resolve(strict=True)
manifest_path = self.root / "manifest.json"
if manifest_path.is_symlink() or not manifest_path.is_file():
raise ObjectDistanceModuleError("LiDAR replay pack identity changed")
manifest = json.loads(manifest_path.read_text())
pack_id = manifest.get("pack_id") if isinstance(manifest, dict) else None
match = _LIDAR_PACK_ID.fullmatch(pack_id) if isinstance(pack_id, str) else None
if (
self.root.is_symlink()
or not self.root.is_dir()
or not isinstance(manifest, dict)
or manifest.get("schema_version") != "missioncore.lidar-replay-pack/v2"
or match is None
or manifest.get("identity_sha256") != match.group(1)
):
raise ObjectDistanceModuleError("LiDAR replay pack identity changed")
artifacts = manifest.get("artifacts")
if not isinstance(artifacts, list):
raise ObjectDistanceModuleError("LiDAR replay pack artifacts changed")
row = next(
(
item
for item in artifacts
if isinstance(item, dict) and item.get("kind") == "lidar-arrays"
),
None,
)
if row is None or row.get("path") != "lidar-replay.npz":
raise ObjectDistanceModuleError("LiDAR replay arrays are unavailable")
arrays_path = self.root / "lidar-replay.npz"
if (
arrays_path.is_symlink()
or not arrays_path.is_file()
or arrays_path.resolve(strict=True).parent != self.root
or arrays_path.stat().st_size != row.get("byte_length")
or _sha(arrays_path) != row.get("sha256")
):
raise ObjectDistanceModuleError("LiDAR replay arrays identity changed")
self.arrays_path = arrays_path.resolve(strict=True)
archive = np.load(arrays_path, allow_pickle=False)
try:
self.arrays = {name: np.asarray(archive[name]) for name in archive.files}
finally:
archive.close()
required = {
"point_offsets",
"point_xyz_map",
"point_received_monotonic_ns",
"pose_positions_map",
"pose_quaternions_map_from_lidar",
"pose_received_monotonic_ns",
}
if not required.issubset(self.arrays):
raise ObjectDistanceModuleError("LiDAR replay array set changed")
self.pack_id = pack_id
self.point_frame_count = int(self.arrays["point_received_monotonic_ns"].shape[0])
self.pose_frame_count = int(self.arrays["pose_received_monotonic_ns"].shape[0])
self.point_count = int(self.arrays["point_xyz_map"].shape[0])
def point_frame(self, index: int) -> _PointFrame:
if not 0 <= index < self.point_frame_count:
raise ObjectDistanceModuleError("LiDAR frame index is outside the replay pack")
begin, end = (int(self.arrays["point_offsets"][index + offset]) for offset in (0, 1))
points = np.asarray(self.arrays["point_xyz_map"][begin:end], dtype=np.float64)
if points.ndim != 2 or points.shape[1:] != (3,) or not np.isfinite(points).all():
raise ObjectDistanceModuleError("LiDAR frame points changed")
return _PointFrame(int(self.arrays["point_received_monotonic_ns"][index]), points)
def pose_frame(self, index: int) -> _PoseFrame:
if not 0 <= index < self.pose_frame_count:
raise ObjectDistanceModuleError("pose frame index is outside the replay pack")
position = tuple(float(value) for value in self.arrays["pose_positions_map"][index])
orientation = tuple(
float(value) for value in self.arrays["pose_quaternions_map_from_lidar"][index]
)
return _PoseFrame(
int(self.arrays["pose_received_monotonic_ns"][index]),
cast(tuple[float, float, float], position),
cast(tuple[float, float, float, float], orientation),
)
def close(self) -> None:
self.arrays.clear()
def _canonical(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
def _sha(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def _projection(path: Path) -> Kb4ProjectionProfile:
candidate = path.expanduser().resolve(strict=True)
# The admitted E10 source pack contains the three small calibration arrays
# together with the full point cloud. Bound the sealed archive itself while
# retaining strict per-member limits for the arrays read below.
if (
candidate.is_symlink()
or not candidate.is_file()
or candidate.stat().st_size > _MAX_CALIBRATION_PACK_BYTES
):
raise ObjectDistanceModuleError("camera/LiDAR calibration is unavailable")
with zipfile.ZipFile(candidate) as archive:
arrays = {}
for name in ("intrinsic_fx_fy_cx_cy", "distortion_kb4", "t_camera_from_lidar"):
info = archive.getinfo(name + ".npy")
if info.file_size > 4096:
raise ObjectDistanceModuleError("camera/LiDAR calibration exceeds its bound")
with archive.open(info) as stream:
arrays[name] = np.lib.format.read_array(stream, allow_pickle=False)
return Kb4ProjectionProfile(
800,
600,
cast(
tuple[float, float, float, float],
tuple(float(value) for value in arrays["intrinsic_fx_fy_cx_cy"]),
),
cast(
tuple[float, float, float, float],
tuple(float(value) for value in arrays["distortion_kb4"]),
),
np.asarray(arrays["t_camera_from_lidar"], dtype=np.float64),
)
def _detections(path: Path, count: int) -> list[dict[str, object]]:
rows: list[dict[str, object]] = []
with path.expanduser().resolve(strict=True).open(encoding="utf-8") as stream:
for raw in stream:
if len(raw) > 8 * 1024 * 1024 or len(rows) >= count:
raise ObjectDistanceModuleError("RF-DETR result exceeds its bound")
row = json.loads(raw)
if not isinstance(row, dict) or row.get("schema_version") != RF_ROW_SCHEMA:
raise ObjectDistanceModuleError("RF-DETR result contract changed")
rows.append(row)
if len(rows) != count:
raise ObjectDistanceModuleError("RF-DETR and LiDAR timelines differ")
return rows
def _integer(value: object, label: str) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise ObjectDistanceModuleError(f"{label} is invalid")
return value
def _number(value: object, label: str) -> float:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ObjectDistanceModuleError(f"{label} is invalid")
number = float(value)
if not math.isfinite(number):
raise ObjectDistanceModuleError(f"{label} is invalid")
return number
def _profile(session_id: str, pack: _LidarPack) -> GeometryProfile:
return GeometryProfile(
profile_id="observatory-object-distance-v1",
provider_id=GEOMETRY_PROVIDER_ID,
source_id="recorded-k1",
session_id=session_id,
source_pack_id=pack.pack_id,
source_pack_sha256=_sha(pack.arrays_path),
frame_count=pack.point_frame_count,
point_count=pack.point_count,
local_surface_model_id="k1-local-surface-shadow-v1",
local_surface_sha256="0" * 64,
valid_frame_count=pack.point_frame_count,
width=800,
height=600,
coordinate_frame="map",
association=ASSOCIATION,
profile_sha256=hashlib.sha256(
_canonical(
{
"module": "object-distance",
"association": ASSOCIATION.__dict__
if hasattr(ASSOCIATION, "__dict__")
else str(ASSOCIATION),
}
)
).hexdigest(),
)
def _packet(
*,
session_id: str,
source_id: str,
frame_id: str,
frame_index: int,
session_seconds: float,
spatial_available: bool,
) -> SourcePacket:
available = ModalityStatus(True, ModalityOutcome.AVAILABLE, "recorded-module-input")
missing = ModalityStatus(False, ModalityOutcome.UNAVAILABLE, "no-synchronous-lidar")
spatial = available if spatial_available else missing
nanoseconds = round(session_seconds * 1_000_000_000)
return SourcePacket(
SourceEnvelope(
source_id=source_id,
session_id=session_id,
frame_id=frame_id,
sequence=frame_index,
timestamps=TimestampBundle(
utc_ns=nanoseconds,
monotonic_ns=nanoseconds,
source_ns=nanoseconds,
clock_basis=ClockBasis.RECORDED_HOST,
),
source_age_ns=0,
binding_reason="camera-lidar-past-only-binding",
calibration_id="camera-1-kb4-05f3ad9b",
representation_id="object-distance-current-cloud-v1",
image=available,
registered_point_increment=spatial,
pose=spatial,
),
b"rf-detr-proposals",
b"current-k1-cloud" if spatial_available else None,
b"current-k1-pose" if spatial_available else None,
)
def _binding_rows(path: Path) -> list[dict[str, object]]:
rows: list[dict[str, object]] = []
previous = -1.0
with path.expanduser().resolve(strict=True).open(encoding="utf-8") as stream:
for raw in stream:
if len(raw) > 64 * 1024 or len(rows) >= 250_000:
raise ObjectDistanceModuleError("LiDAR binding timeline exceeds its bound")
row = json.loads(raw)
if (
not isinstance(row, dict)
or row.get("schema_version") != "missioncore.m49-tgs-portable-source-index-row/v1"
or row.get("timeline_frame_index") != len(rows)
):
raise ObjectDistanceModuleError("LiDAR binding row identity changed")
seconds = _number(row.get("session_seconds"), "LiDAR binding time")
if seconds <= previous or not isinstance(row.get("sample_available"), bool):
raise ObjectDistanceModuleError("LiDAR binding timeline changed")
rows.append(row)
previous = seconds
if not rows:
raise ObjectDistanceModuleError("LiDAR binding timeline is empty")
return rows
def _aligned_camera_seconds(
binding: dict[str, object],
rf_row: dict[str, object],
*,
frame_index: int,
previous_camera_seconds: float,
) -> float:
# Validate both clocks, but join the sealed products by their shared frame
# index. Camera-source repairs MP4 discontinuities; the LiDAR binding keeps
# the original segment clock, so equality between their seconds is invalid.
_number(binding.get("session_seconds"), "LiDAR binding time")
if rf_row.get("frame_index") != frame_index:
raise ObjectDistanceModuleError("RF-DETR and LiDAR frame identities differ")
seconds = _number(rf_row.get("session_seconds"), "camera session time")
if seconds <= previous_camera_seconds:
raise ObjectDistanceModuleError("RF-DETR camera timeline changed")
return seconds
def execute(
*,
detections: Path,
lidar_pack: Path,
binding_index: Path,
calibration: Path,
output: Path,
session_id: str,
) -> dict[str, object]:
index = _binding_rows(binding_index)
rf_rows = _detections(detections, len(index))
projection = _projection(calibration)
output = output.expanduser().absolute()
if output.exists():
if output.is_symlink() or not output.is_dir() or any(output.iterdir()):
raise ObjectDistanceModuleError("output root is unsafe")
else:
output.mkdir(mode=0o700, parents=True, exist_ok=False)
if output.is_symlink() or not output.is_dir():
raise ObjectDistanceModuleError("output root is unsafe")
pack = _LidarPack(lidar_pack)
started = time.monotonic()
ranged = 0
proposal_count = 0
unavailable = 0
result_path = output / "object-distances.jsonl"
try:
store = _CurrentStore(_profile(session_id, pack))
provider = Ravnoves00GeometryAssociationProvider(store=store) # type: ignore[arg-type]
surface = K1LocalSurfaceShadowEstimator()
previous_camera_seconds = -1.0
with result_path.open("xb") as stream:
for frame_index, (binding, rf_row) in enumerate(zip(index, rf_rows, strict=True)):
seconds = _aligned_camera_seconds(
binding,
rf_row,
frame_index=frame_index,
previous_camera_seconds=previous_camera_seconds,
)
previous_camera_seconds = seconds
raw_proposals = rf_row.get("proposals")
if not isinstance(raw_proposals, list):
raise ObjectDistanceModuleError("RF-DETR proposals are unavailable")
proposals = tuple(ObjectProposal2D.from_dict(value) for value in raw_proposals)
proposal_count += len(proposals)
store.current = None
available = binding["sample_available"] is True
if available:
point = pack.point_frame(
_integer(binding["selected_lidar_frame_index"], "LiDAR frame index")
)
pose = pack.pose_frame(
_integer(binding["selected_pose_frame_index"], "pose frame index")
)
age_ms = abs(point.received_monotonic_ns - pose.received_monotonic_ns) / 1e6
surface_frame = surface.process(
K1LocalSurfaceShadowInput(
frame_index=frame_index,
source_frame_index=_integer(
binding["source_frame_index"], "source frame index"
),
session_seconds=seconds,
pose_binding_age_ms=age_ms,
points_map=point.xyz_map,
position_map=np.asarray(pose.position_map, dtype=np.float64),
published_monotonic_ns=point.received_monotonic_ns,
)
)
store.current = GeometryFrame(
frame_index,
point.xyz_map,
surface_frame.point_class,
np.asarray(pose.position_map, dtype=np.float64),
np.asarray(pose.orientation_map_from_lidar, dtype=np.float64),
projection,
surface_frame.valid,
)
elif proposals:
unavailable += len(proposals)
source_id = proposals[0].source_id if proposals else "recorded-k1"
frame_id = proposals[0].frame_id if proposals else f"frame-{frame_index + 1:06d}"
observations = tuple(
item
for item in provider.associate(
_packet(
session_id=session_id,
source_id=source_id,
frame_id=frame_id,
frame_index=frame_index,
session_seconds=seconds,
spatial_available=available,
),
proposals,
)
if item.proposal_ids
)
ranged += sum(item.metric_geometry is not None for item in observations)
stream.write(
_canonical(
{
"schema_version": ROW_SCHEMA,
"frame_index": frame_index,
"session_seconds": seconds,
"observations": [item.to_dict() for item in observations],
}
)
+ b"\n"
)
result = {
"schema_version": SCHEMA,
"module_id": "object-distance",
"source_session_id": session_id,
"frame_count": len(index),
"proposal_count": proposal_count,
"ranged_proposal_count": ranged,
"unavailable_proposal_count": unavailable,
"object_distances_sha256": _sha(result_path),
"elapsed_seconds": time.monotonic() - started,
"range_estimator": "median-camera-z-of-owned-current-points/v1",
"authority": AUTHORITY,
}
(output / "result.json").write_bytes(_canonical(result))
return result
finally:
pack.close()
def _package_session(path: Path) -> str:
document = json.loads(path.read_text())
source = document.get("source") if isinstance(document, dict) else None
session_id = source.get("session_id") if isinstance(source, dict) else None
if (
not isinstance(document, dict)
or document.get("schema_version") != "missioncore.observatory-portable-lab-v1-source/v1"
or not isinstance(session_id, str)
):
raise ObjectDistanceModuleError("prepared source identity changed")
return session_id
def main(argv: list[str] | None = None) -> int:
arguments = list(sys.argv[1:] if argv is None else argv)
if arguments == ["--package-step", "object-distance"]:
execute(
detections=_PACKAGE_DETECTIONS,
lidar_pack=_PACKAGE_LIDAR_PACK,
binding_index=_PACKAGE_BINDING_INDEX,
calibration=_PACKAGE_CALIBRATION,
output=_PACKAGE_OUTPUT,
session_id=_package_session(_PACKAGE_SOURCE),
)
return 0
parser = argparse.ArgumentParser()
parser.add_argument("--detections", type=Path, required=True)
parser.add_argument("--lidar-pack", type=Path, required=True)
parser.add_argument("--binding-index", type=Path, required=True)
parser.add_argument("--calibration", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--session-id", required=True)
execute(**vars(parser.parse_args(arguments)))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,310 @@
#!/usr/bin/env python3
"""Recorded-camera RF-DETR module with a sealed, path-local output contract."""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
import sys
import time
from pathlib import Path
import numpy as np
from PIL import Image
from k1link.perception.contracts import (
ClockBasis,
ModalityOutcome,
ModalityStatus,
SourceEnvelope,
TimestampBundle,
)
from k1link.perception.detector import NativeRfDetrShadowDetectorProvider
from k1link.perception.providers import SourcePacket
from k1link.perception.rf_detr_native_object_detector import (
RF_DETR_NATIVE_ENGINE_SHA256,
TritonNativeRfDetrHttpInferenceBackend,
)
from k1link.perception.yolox_object_detector import load_valid_fov_mask
SCHEMA = "missioncore.observatory-ai-module-rf-detr-result/v1"
ROW_SCHEMA = "missioncore.observatory-ai-module-rf-detr-frame/v1"
AUTHORITY = {
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
}
VALID_FOV_SHA256 = "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63"
_PACKAGE_SOURCE = Path("/missioncore/input/steps/prepare/source-input.json")
_PACKAGE_FRAMES = Path("/missioncore/input/steps/camera-source/source-frames")
_PACKAGE_TIMELINE = Path("/missioncore/input/steps/camera-source/timeline.jsonl")
_PACKAGE_VALID_FOV = Path("/opt/nodedc/assets/valid-fov-mask")
_PACKAGE_ENGINE = Path("/models/rf_detr_large_native_kb4/1/model.plan")
_PACKAGE_OUTPUT = Path("/missioncore/output")
class RfDetrModuleError(RuntimeError):
pass
def _canonical(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
def _sha(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def _file(path: Path, expected: str, label: str) -> Path:
candidate = path.expanduser().absolute()
if candidate.is_symlink():
raise RfDetrModuleError(f"{label} identity changed")
candidate = candidate.resolve(strict=True)
if not candidate.is_file() or _sha(candidate) != expected:
raise RfDetrModuleError(f"{label} identity changed")
return candidate
def _empty(path: Path) -> Path:
candidate = path.expanduser().absolute()
if candidate.exists():
if candidate.is_symlink() or not candidate.is_dir() or any(candidate.iterdir()):
raise RfDetrModuleError("output root is unsafe")
else:
candidate.mkdir(mode=0o700, parents=True, exist_ok=False)
if candidate.is_symlink() or not candidate.is_dir():
raise RfDetrModuleError("output root is unsafe")
return candidate
def _rows(path: Path, *, maximum: int) -> list[dict[str, object]]:
result: list[dict[str, object]] = []
with path.open(encoding="utf-8") as stream:
for raw in stream:
if len(raw) > 64 * 1024 or len(result) >= maximum:
raise RfDetrModuleError("camera timeline exceeds the module bound")
row = json.loads(raw)
if not isinstance(row, dict):
raise RfDetrModuleError("camera timeline row is invalid")
result.append(row)
if not result:
raise RfDetrModuleError("camera timeline is empty")
return result
def _wait_triton(process: subprocess.Popen[bytes]) -> None:
import http.client
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
if process.poll() is not None:
raise RfDetrModuleError("RF-DETR inference runtime stopped during startup")
try:
connection = http.client.HTTPConnection("127.0.0.1", 8000, timeout=1)
connection.request("GET", "/v2/models/rf_detr_large_native_kb4/ready")
response = connection.getresponse()
response.read()
connection.close()
if response.status == 200:
return
except OSError:
pass
time.sleep(0.1)
raise RfDetrModuleError("RF-DETR inference runtime did not become ready")
def _packet(
session_id: str, source_id: str, frame_index: int, session_seconds: float, image: np.ndarray
) -> SourcePacket:
available = ModalityStatus(True, ModalityOutcome.AVAILABLE, "recorded-camera-frame")
unavailable = ModalityStatus(False, ModalityOutcome.UNAVAILABLE, "module-input-not-requested")
nanoseconds = round(session_seconds * 1_000_000_000)
envelope = SourceEnvelope(
source_id=source_id,
session_id=session_id,
frame_id=f"frame-{frame_index + 1:06d}",
sequence=frame_index,
timestamps=TimestampBundle(
utc_ns=nanoseconds,
monotonic_ns=nanoseconds,
source_ns=nanoseconds,
clock_basis=ClockBasis.RECORDED_HOST,
),
source_age_ns=0,
binding_reason="recorded-camera-timeline",
calibration_id="camera-1-kb4-05f3ad9b",
representation_id="rf-detr-native-kb4-v1",
image=available,
registered_point_increment=unavailable,
pose=unavailable,
)
return SourcePacket(envelope, image, None, None)
def execute(
*,
frames: Path,
timeline: Path,
valid_fov: Path,
engine: Path,
output: Path,
session_id: str,
source_id: str,
) -> dict[str, object]:
frames = frames.expanduser().resolve(strict=True)
if frames.is_symlink() or not frames.is_dir():
raise RfDetrModuleError("prepared camera frames are unavailable")
timeline = timeline.expanduser().resolve(strict=True)
valid_fov = _file(valid_fov, VALID_FOV_SHA256, "valid-FOV mask")
_file(engine, RF_DETR_NATIVE_ENGINE_SHA256, "RF-DETR TensorRT engine")
rows = _rows(timeline, maximum=100_000)
names = tuple(f"frame-{index + 1:06d}.png" for index in range(len(rows)))
if tuple(sorted(path.name for path in frames.iterdir())) != names:
raise RfDetrModuleError("prepared camera frame set changed")
output = _empty(output)
log = (output / "triton.log").open("wb")
process = subprocess.Popen(
[
"tritonserver",
"--model-repository=/models",
"--model-control-mode=explicit",
"--load-model=rf_detr_large_native_kb4",
"--allow-grpc=false",
"--allow-metrics=false",
"--http-address=127.0.0.1",
"--pinned-memory-pool-byte-size=16777216",
"--cuda-memory-pool-byte-size=0:16777216",
],
stdout=log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
backend: TritonNativeRfDetrHttpInferenceBackend | None = None
started = time.monotonic()
counts = 0
detections_path = output / "detections.jsonl"
try:
_wait_triton(process)
backend = TritonNativeRfDetrHttpInferenceBackend("http://127.0.0.1:8000")
detector = NativeRfDetrShadowDetectorProvider(
mask=load_valid_fov_mask(valid_fov),
backend=backend,
)
detector.warm_up()
with detections_path.open("xb") as stream:
previous = -1.0
for frame_index, (name, row) in enumerate(zip(names, rows, strict=True)):
seconds = row.get("session_seconds")
if (
row.get("frame_index") != frame_index
or isinstance(seconds, bool)
or not isinstance(seconds, (int, float))
or float(seconds) <= previous
):
raise RfDetrModuleError("camera timeline identity changed")
with Image.open(frames / name) as source:
rgb = np.asarray(source.convert("RGB"), dtype=np.uint8)
if rgb.shape != (600, 800, 3):
raise RfDetrModuleError("camera raster changed")
proposals = detector.detect(
_packet(
session_id,
source_id,
frame_index,
float(seconds),
np.ascontiguousarray(rgb[:, :, ::-1]),
)
)
counts += len(proposals)
stream.write(
_canonical(
{
"schema_version": ROW_SCHEMA,
"frame_index": frame_index,
"session_seconds": float(seconds),
"proposals": [proposal.to_dict() for proposal in proposals],
}
)
+ b"\n"
)
previous = float(seconds)
snapshot = detector.snapshot()
result = {
"schema_version": SCHEMA,
"module_id": "rf-detr",
"source": {"session_id": session_id, "source_id": source_id},
"frame_count": len(rows),
"proposal_count": counts,
"zero_proposal_frame_count": snapshot.zero_proposal_frames,
"detections_sha256": _sha(detections_path),
"elapsed_seconds": time.monotonic() - started,
"authority": AUTHORITY,
}
(output / "result.json").write_bytes(_canonical(result))
return result
finally:
if backend is not None:
backend.close()
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=5)
log.close()
def _package_source(path: Path) -> tuple[str, str]:
document = json.loads(path.read_text())
if (
not isinstance(document, dict)
or document.get("schema_version") != "missioncore.observatory-portable-lab-v1-source/v1"
or not isinstance(document.get("source"), dict)
or not isinstance(document.get("camera_compute_job"), dict)
):
raise RfDetrModuleError("prepared camera source contract changed")
source = document["source"]
camera = document["camera_compute_job"]
session_id = source.get("session_id")
source_id = camera.get("source_id")
if not isinstance(session_id, str) or not isinstance(source_id, str):
raise RfDetrModuleError("prepared camera source identity changed")
return session_id, source_id
def main(argv: list[str] | None = None) -> int:
arguments = list(sys.argv[1:] if argv is None else argv)
if arguments == ["--package-step", "rf-detr"]:
session_id, source_id = _package_source(_PACKAGE_SOURCE)
execute(
frames=_PACKAGE_FRAMES,
timeline=_PACKAGE_TIMELINE,
valid_fov=_PACKAGE_VALID_FOV,
engine=_PACKAGE_ENGINE,
output=_PACKAGE_OUTPUT,
session_id=session_id,
source_id=source_id,
)
return 0
parser = argparse.ArgumentParser()
parser.add_argument("--frames", type=Path, required=True)
parser.add_argument("--timeline", type=Path, required=True)
parser.add_argument("--valid-fov", type=Path, required=True)
parser.add_argument("--engine", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--session-id", required=True)
parser.add_argument("--source-id", required=True)
options = parser.parse_args(arguments)
execute(**vars(options))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""Decode one sealed K1 camera epoch without loading any AI model."""
from __future__ import annotations
import os
import shutil
import sys
from pathlib import Path
from portable_lab_v1_component_adapter import (
AssetExpectation,
ComponentAdapterError,
RuntimeLayout,
load_component_request,
resolve_runtime_layout,
run_command,
sha256_file,
validate_camera_compute_job,
validate_tree_asset,
validate_tree_receipt,
)
from run_portable_lab_v1_eomt_component import (
FFMPEG_TREE_BYTE_LENGTH,
FFMPEG_TREE_IDENTITY_SHA256,
FFMPEG_TREE_SOURCE_IMAGE_SHA256,
_decode_camera_epoch,
_source_frame_manifest_document,
_write_canonical_json,
)
FFMPEG_ASSET = AssetExpectation(
"eomt-ffmpeg-runtime",
"/opt/nodedc/assets/ffmpeg-runtime",
"tree",
"identity-sha256",
FFMPEG_TREE_IDENTITY_SHA256,
FFMPEG_TREE_BYTE_LENGTH,
)
def execute(*, request_path: Path, layout: RuntimeLayout) -> None:
request = load_component_request(
request_path, component="camera-source", expectations=(FFMPEG_ASSET,)
)
output = layout.output_root
if output.is_symlink() or not output.is_dir() or any(output.iterdir()):
raise ComponentAdapterError("camera-source output must be an empty real directory")
input_document = validate_camera_compute_job(layout.camera_job_root, request.source)
ffmpeg_root = validate_tree_asset(request, layout, FFMPEG_ASSET.asset_id)
validate_tree_receipt(
request,
ffmpeg_root,
FFMPEG_ASSET.asset_id,
expected_metadata={
"source_image_sha256": FFMPEG_TREE_SOURCE_IMAGE_SHA256,
"source_path": "/usr/lib/ffmpeg/7.0",
},
additional_metadata_keys=frozenset({"binaries"}),
verify_payload=True,
)
workspace = output / ".camera-source-work"
workspace.mkdir(mode=0o700)
try:
frames, timeline, repair = _decode_camera_epoch(
request=request,
input_document=input_document,
camera_job_root=layout.camera_job_root,
output_root=output,
work_root=workspace,
ffmpeg_root=ffmpeg_root,
command_runner=run_command,
)
_write_canonical_json(
output / "source-frames.json", _source_frame_manifest_document(frames, request)
)
_write_canonical_json(output / "decode-repair.json", repair)
os.replace(timeline, output / "timeline.jsonl")
_write_canonical_json(
output / "camera-source.json",
{
"schema_version": "missioncore.observatory-prepared-camera/v1",
"source": {
"camera_job_id": request.source.camera_job_id,
"input_sha256": request.source.camera_input_sha256,
"frame_count": request.source.frame_count,
},
"artifacts": [
{
"role": role,
"path": name,
"byte_length": (output / name).stat().st_size,
"sha256": sha256_file(output / name),
}
for role, name in (
("decode-repair", "decode-repair.json"),
("source-frames", "source-frames.json"),
("timeline", "timeline.jsonl"),
)
],
},
)
finally:
shutil.rmtree(workspace, ignore_errors=True)
def main() -> int:
layout = resolve_runtime_layout(
tuple(sys.argv[1:]), component="camera-source", expectations=(FFMPEG_ASSET,)
)
execute(request_path=layout.request, layout=layout)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Run the sealed LAB V1 DDRNet component over EoMT-decoded K1 frames."""
"""Run DDRNet over independently prepared, immutable K1 camera frames."""
from __future__ import annotations
@@ -38,6 +38,7 @@ from portable_lab_v1_component_adapter import (
validate_file_asset,
validate_fixed_result,
validate_grayscale_png_payload,
validate_prepared_camera_root,
)
DDRNET_RESULT_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-run/v1"
@@ -136,7 +137,7 @@ def execute_ddrnet_component(
layout.effective_ddrnet_config,
request,
)
eomt_root, source_frames, decode_repair = _validate_eomt_input(
prepared_root, source_frames, decode_repair = _validate_prepared_input(
layout.eomt_result_root,
request,
)
@@ -148,7 +149,7 @@ def execute_ddrnet_component(
# tmpfs makes a full 6,830-frame K1 run fail even though the immutable
# input is valid. Verify every manifest digest through an O_NOFOLLOW
# descriptor, then let the sealed runner read that same read-only tree.
frames_root = _verify_source_frames(eomt_root, source_frames)
frames_root = _verify_source_frames(prepared_root, source_frames)
mapping_copy = workspace / "goose_label_mapping.csv"
shutil.copyfile(assets["ddrnet-goose-mapping"], mapping_copy)
os.chmod(mapping_copy, 0o400)
@@ -182,7 +183,7 @@ def execute_ddrnet_component(
"0",
)
command_runner(argv, _ddrnet_environment())
shutil.copyfile(eomt_root / "decode-repair.json", staging / "decode-repair.json")
shutil.copyfile(prepared_root / "decode-repair.json", staging / "decode-repair.json")
if (
load_json(
staging / "decode-repair.json",
@@ -269,13 +270,29 @@ def _validate_effective_config(
return config
def _validate_eomt_input(
def _validate_prepared_input(
root: Path,
request: ComponentRequest,
) -> tuple[Path, tuple[SourceFrameRow, ...], dict[str, object]]:
if root.is_symlink():
raise ComponentAdapterError("EoMT result root is a symbolic link")
raise ComponentAdapterError("prepared camera root is a symbolic link")
resolved = root.resolve(strict=True)
if not (resolved / "result.json").exists():
validate_prepared_camera_root(resolved, request)
manifest_path = _confined_regular_file(
resolved / "source-frames.json", resolved, "prepared source frame manifest"
)
source_frames = _source_frame_manifest(manifest_path, resolved, request)
repair = load_json(
resolved / "decode-repair.json",
label="camera decode repair",
maximum=1024 * 1024,
confinement_root=resolved,
)
_validate_decode_repair(repair, request.source.frame_count)
_confined_regular_file(resolved / "timeline.jsonl", resolved, "prepared camera timeline")
return resolved, source_frames, repair
# Migration compatibility for already sealed dual-model releases only.
result = validate_fixed_result(
resolved,
schema_version=EOMT_RESULT_SCHEMA,
@@ -37,6 +37,7 @@ from portable_lab_v1_component_adapter import (
validate_fixed_result,
validate_grayscale_png_payload,
validate_identity_manifest,
validate_prepared_camera_root,
validate_tree_asset,
validate_tree_receipt,
)
@@ -50,11 +51,10 @@ MODEL_REVISION: Final = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f"
MODEL_ID: Final = "tue-mps/cityscapes_semantic_eomt_large_1024"
MODEL_ARCHITECTURE: Final = "EomtForUniversalSegmentation"
PHYSICAL_CAMERA_SOURCE_ID: Final = "sensor.camera.right"
# Keep a large post-run floor while admitting the full 6,830-frame K1 record on
# Worker 006. The independent ``reserve`` below already accounts for the
# complete worst-case working set, so adding the historical 360 GiB floor made
# the real job miss admission by about 1.4 GB despite 408 GB being free.
DISK_FLOOR_BYTES: Final = 350 * 1024**3
# Owner-approved post-run disk floor (2026-09-03); not a RAM/VRAM reservation.
# The independent ``reserve`` below remains additional to this free-space floor.
# Deploy only through a new sealed image/release, never patch an installed digest.
DISK_FLOOR_BYTES: Final = 250 * 1024**3
FFMPEG_TREE_SOURCE_IMAGE_SHA256: Final = (
"8a364092b03561b9c08ac00730206e363a53d07ea0304f7d543b403b65432b5e"
)
@@ -163,26 +163,49 @@ def execute_eomt_component(
component="eomt",
expectations=EOMT_ASSETS,
)
input_document = validate_camera_compute_job(layout.camera_job_root, request.source)
roots = _validate_release_assets(request, layout)
output = _empty_output_root(layout.output_root)
reserve = request.source.frame_count * 800 * 600 * 7 + request.source.input_byte_length
free_before = available_bytes(output)
if disk_floor_bytes < 0 or free_before < disk_floor_bytes + reserve:
if (
isinstance(disk_floor_bytes, bool)
or not isinstance(disk_floor_bytes, int)
or disk_floor_bytes < 0
or free_before < disk_floor_bytes + reserve
):
raise ComponentAdapterError("EoMT output does not satisfy its disk reserve")
# Fail before hashing the complete recording and several GiB of model assets.
# The typed request supplies a bounded estimate; input/asset validation is
# still mandatory before decoding or model execution.
input_document = validate_camera_compute_job(layout.camera_job_root, request.source)
roots = _validate_release_assets(request, layout)
workspace = _prepare_workspace(output / ".eomt-work" if work_root is None else work_root)
try:
total_started = time.perf_counter()
extract_started = time.perf_counter()
frames_root, timeline_path, decode_repair = _decode_camera_epoch(
request=request,
input_document=input_document,
camera_job_root=layout.camera_job_root,
output_root=output,
work_root=workspace,
ffmpeg_root=roots["eomt-ffmpeg-runtime"],
command_runner=command_runner,
)
if (layout.eomt_result_root / "camera-source.json").is_file():
prepared = validate_prepared_camera_root(layout.eomt_result_root, request)
frames_root = layout.eomt_result_root / "source-frames"
timeline_path = prepared["timeline"]
decode_repair = load_json(
prepared["decode-repair"],
label="prepared camera decode repair",
maximum=1024 * 1024,
confinement_root=layout.eomt_result_root,
)
_validate_source_frame_manifest(prepared["source-frames"], frames_root, request)
else:
# Historical fixed-layout tests and already sealed releases retain
# their old in-component decode path. New modular packages always
# mount camera-source and never make one model prepare another.
frames_root, timeline_path, decode_repair = _decode_camera_epoch(
request=request,
input_document=input_document,
camera_job_root=layout.camera_job_root,
output_root=output,
work_root=workspace,
ffmpeg_root=roots["eomt-ffmpeg-runtime"],
command_runner=command_runner,
)
extract_seconds = _elapsed(extract_started)
free_post_extract = available_bytes(output)
if free_post_extract < disk_floor_bytes: