feat(observatory): add portable calculation profiles
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,836 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import k1link.observatory.m49_portable_source as source_module
|
||||
from k1link.observatory.m49_portable_executor import (
|
||||
M49_PORTABLE_RUNTIME_PHASES,
|
||||
M49PortableBoundSourceStage,
|
||||
M49PortableProfileRunnerAdapter,
|
||||
M49PortableRunnerInstallation,
|
||||
M49PortableSourceMaterializerAdapter,
|
||||
)
|
||||
from k1link.observatory.m49_portable_result import (
|
||||
M49PortableResultError,
|
||||
validate_m49_portable_result,
|
||||
)
|
||||
from k1link.observatory.m49_portable_source import (
|
||||
M49PortableSourceError,
|
||||
M49PortableSourceIdentity,
|
||||
materialize_m49_portable_source,
|
||||
read_m49_source_index,
|
||||
validate_m49_portable_source_stage,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
PortableResultPackageManifest,
|
||||
PortableResultValidationContext,
|
||||
canonical_json,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerRuntimePlan,
|
||||
PortableWorkerSourceStage,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJob,
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.source_admission import (
|
||||
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
)
|
||||
from k1link.observatory.worker_agent import (
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
SealedObservatoryRecordedJob,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
PROFILE_PATH = REPOSITORY_ROOT / "config" / "perception" / "m49-tgs-portable-v2.json"
|
||||
RUNNER_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
/ "run_m49_tgs_portable.cpp"
|
||||
)
|
||||
BUILDER_PATH = REPOSITORY_ROOT / "scripts" / "build_m49_portable_executor_release.py"
|
||||
DOCKERFILE_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
/ "Dockerfile.m49-portable-executor"
|
||||
)
|
||||
INSTALLER_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
/ "Invoke-M49PortableExecutorCandidateInstall.ps1"
|
||||
)
|
||||
NOW = "2026-08-31T09:00:00.000Z"
|
||||
SESSION_ID = "20260831T085500Z_viewer_live"
|
||||
AUTHORITY = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
RAW_PAYLOAD = b"exact-k1-raw-replay"
|
||||
METADATA_PAYLOAD = b'{"received_monotonic_ns":1}\n'
|
||||
|
||||
_builder_spec = importlib.util.spec_from_file_location(
|
||||
"build_m49_portable_executor_release", BUILDER_PATH
|
||||
)
|
||||
assert _builder_spec is not None and _builder_spec.loader is not None
|
||||
builder = importlib.util.module_from_spec(_builder_spec)
|
||||
sys.modules[_builder_spec.name] = builder
|
||||
_builder_spec.loader.exec_module(builder)
|
||||
|
||||
|
||||
class _FakeLidarPack:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
adapter_sha256: str,
|
||||
raw_sha256: str,
|
||||
metadata_sha256: str,
|
||||
) -> None:
|
||||
del adapter_sha256
|
||||
self.pack_id = f"lidar-replay-pack-{'c' * 64}"
|
||||
self.identity = {
|
||||
"session_id": SESSION_ID,
|
||||
"logical_content_sha256": "e" * 64,
|
||||
"source_evidence": {
|
||||
"raw": {
|
||||
"byte_length": len(RAW_PAYLOAD),
|
||||
"sha256": raw_sha256,
|
||||
},
|
||||
"metadata": {
|
||||
"byte_length": len(METADATA_PAYLOAD),
|
||||
"sha256": metadata_sha256,
|
||||
},
|
||||
},
|
||||
}
|
||||
self.manifest = {"identity_sha256": "d" * 64}
|
||||
self.arrays = {
|
||||
"point_received_monotonic_ns": np.asarray([100_000_000, 1_100_000_000], dtype=np.int64),
|
||||
"pose_received_monotonic_ns": np.asarray([50_000_000, 1_050_000_000], dtype=np.int64),
|
||||
"pose_positions_map": np.zeros((2, 3), dtype=np.float64),
|
||||
}
|
||||
self._points = (
|
||||
np.asarray(
|
||||
[[2.0, 0.0, 0.0], [3.0, 0.0, 1.0], [4.0, 0.0, 0.2]],
|
||||
dtype=np.float64,
|
||||
),
|
||||
np.asarray(
|
||||
[[2.5, 0.0, 0.0], [3.5, 0.0, 1.0], [4.5, 0.0, 0.2]],
|
||||
dtype=np.float64,
|
||||
),
|
||||
)
|
||||
|
||||
def point_frame(self, index: int) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
xyz_map=self._points[index],
|
||||
intensity=np.asarray([10, 20, 30], dtype=np.uint8),
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _m49_definition() -> PortableRunDefinition:
|
||||
return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).resolve_setup(
|
||||
"m49-tgs-portable-v2"
|
||||
)
|
||||
|
||||
|
||||
def _materialized_source(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
include_metadata_member: bool = True,
|
||||
) -> tuple[Path, M49PortableSourceIdentity]:
|
||||
definition = _m49_definition()
|
||||
adapter_sha256 = definition.source_adapter.contract_sha256
|
||||
raw_sha256 = hashlib.sha256(RAW_PAYLOAD).hexdigest()
|
||||
metadata_sha256 = hashlib.sha256(METADATA_PAYLOAD).hexdigest()
|
||||
bundle = {
|
||||
"schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
"source_session_id": SESSION_ID,
|
||||
"source_catalog_sha256": "1" * 64,
|
||||
"plugin_id": definition.source_requirements.plugin_id,
|
||||
"archive_id": definition.source_requirements.archive_id,
|
||||
"source_adapter": {
|
||||
"id": definition.source_adapter.adapter_id,
|
||||
"version": definition.source_adapter.version,
|
||||
"sha256": adapter_sha256,
|
||||
},
|
||||
"sources": [],
|
||||
"spatial_replay": {
|
||||
"timeline_origin_monotonic_ns": 0,
|
||||
"members": [
|
||||
{
|
||||
"artifact_id": "raw-transport-primary",
|
||||
"media_type": "application/x-nodedc-k1mqtt",
|
||||
"byte_length": len(RAW_PAYLOAD),
|
||||
"replay_byte_length": len(RAW_PAYLOAD),
|
||||
"sha256": raw_sha256,
|
||||
},
|
||||
*(
|
||||
[
|
||||
{
|
||||
"artifact_id": "raw-transport-index",
|
||||
"media_type": "application/x-ndjson",
|
||||
"byte_length": len(METADATA_PAYLOAD),
|
||||
"replay_byte_length": len(METADATA_PAYLOAD),
|
||||
"sha256": metadata_sha256,
|
||||
}
|
||||
]
|
||||
if include_metadata_member
|
||||
else []
|
||||
),
|
||||
],
|
||||
},
|
||||
"camera": {
|
||||
"generation_sha256": "2" * 64,
|
||||
"epoch": {
|
||||
"timeline_start_seconds": 0.0,
|
||||
"timeline_end_seconds": 3.0,
|
||||
"segments": [
|
||||
{"sequence": 1, "end_time_seconds": 1.0},
|
||||
{"sequence": 2, "end_time_seconds": 2.0},
|
||||
{"sequence": 3, "end_time_seconds": 3.0},
|
||||
],
|
||||
},
|
||||
},
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
bundle_payload = canonical_json(bundle)
|
||||
bundle_sha256 = hashlib.sha256(bundle_payload).hexdigest()
|
||||
capability = {
|
||||
"schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
"source_session_id": SESSION_ID,
|
||||
"source_catalog_sha256": "1" * 64,
|
||||
"source_bundle_sha256": bundle_sha256,
|
||||
"source_adapter_sha256": adapter_sha256,
|
||||
"modalities": ["point-cloud", "trajectory", "video"],
|
||||
"camera_profile": {
|
||||
"generation_sha256": "2" * 64,
|
||||
"frame_count": 3,
|
||||
},
|
||||
"calibration": {},
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
capability_payload = canonical_json(capability)
|
||||
capability_sha256 = hashlib.sha256(capability_payload).hexdigest()
|
||||
bundle_path = tmp_path / "source-bundle.json"
|
||||
capability_path = tmp_path / "source-capability.json"
|
||||
bundle_path.write_bytes(bundle_payload)
|
||||
capability_path.write_bytes(capability_payload)
|
||||
expected = M49PortableSourceIdentity(
|
||||
source_session_id=SESSION_ID,
|
||||
source_catalog_sha256="1" * 64,
|
||||
source_bundle_sha256=bundle_sha256,
|
||||
source_capability_manifest_sha256=capability_sha256,
|
||||
source_adapter_sha256=adapter_sha256,
|
||||
raw_capture_sha256=raw_sha256,
|
||||
metadata_sha256=metadata_sha256,
|
||||
)
|
||||
fake = _FakeLidarPack(
|
||||
adapter_sha256=adapter_sha256,
|
||||
raw_sha256=raw_sha256,
|
||||
metadata_sha256=metadata_sha256,
|
||||
)
|
||||
monkeypatch.setattr(source_module, "LidarReplayPackV2", lambda _root: fake)
|
||||
stage = materialize_m49_portable_source(
|
||||
source_bundle_path=bundle_path,
|
||||
source_capability_path=capability_path,
|
||||
lidar_pack_root=tmp_path,
|
||||
profile_path=PROFILE_PATH,
|
||||
output_parent=tmp_path / "source-stages",
|
||||
expected=expected,
|
||||
)
|
||||
return stage.root, expected
|
||||
|
||||
|
||||
def _ready_m49_registry(tmp_path: Path) -> PortableRunDefinitionRegistry:
|
||||
base_registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
base = base_registry.resolve_setup("m49-tgs-portable-v2")
|
||||
document = cast(dict[str, object], json.loads(REGISTRY_PATH.read_text(encoding="utf-8")))
|
||||
selected = copy.deepcopy(
|
||||
next(
|
||||
cast(dict[str, object], row)
|
||||
for row in cast(list[object], document["definitions"])
|
||||
if cast(dict[str, object], row)["setup_id"] == "m49-tgs-portable-v2"
|
||||
)
|
||||
)
|
||||
runner = {
|
||||
"component_id": "m49-tgs-portable-runner-v1",
|
||||
"kind": "runner",
|
||||
"sha256": _sha256(RUNNER_PATH),
|
||||
}
|
||||
components = cast(list[object], selected["components"])
|
||||
components.append(runner)
|
||||
components.sort(key=lambda value: cast(str, cast(dict[str, object], value)["component_id"]))
|
||||
executor = {
|
||||
"contour_id": "worker-006",
|
||||
"state": "ready",
|
||||
"release_id": "m49-tgs-portable-executor-v1",
|
||||
"release_sha256": "3" * 64,
|
||||
"image_sha256": "4" * 64,
|
||||
"reason_code": None,
|
||||
"reason": None,
|
||||
}
|
||||
selected["executor"] = executor
|
||||
identity = copy.deepcopy(base.identity_document())
|
||||
identity["components"] = copy.deepcopy(components)
|
||||
identity["executor"] = {
|
||||
key: executor[key]
|
||||
for key in ("contour_id", "state", "release_id", "release_sha256", "image_sha256")
|
||||
}
|
||||
selected["definition_sha256"] = canonical_sha256(identity)
|
||||
path = tmp_path / "ready-m49-registry.json"
|
||||
path.write_bytes(
|
||||
canonical_json(
|
||||
{
|
||||
"schema_version": document["schema_version"],
|
||||
"definitions": [selected],
|
||||
}
|
||||
)
|
||||
)
|
||||
return PortableRunDefinitionRegistry.from_file(path)
|
||||
|
||||
|
||||
def _running_job(
|
||||
tmp_path: Path,
|
||||
definition: PortableRunDefinition,
|
||||
expected: M49PortableSourceIdentity,
|
||||
) -> tuple[ObservatoryRecordedJobQueue, ObservatoryRecordedJob, str]:
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path / "queue",
|
||||
definitions=RecordedRunDefinitionRegistry((definition.to_recorded_run_definition(),)),
|
||||
clock=lambda: NOW,
|
||||
)
|
||||
job, created = queue.submit(
|
||||
ObservatoryRecordedJobIntent(
|
||||
idempotency_key="m49-portable-executor-test-001",
|
||||
source_session_id=SESSION_ID,
|
||||
source_catalog_sha256=expected.source_catalog_sha256,
|
||||
source_bundle_sha256=expected.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=(expected.source_capability_manifest_sha256),
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
),
|
||||
enqueue=True,
|
||||
)
|
||||
assert created is True
|
||||
claim = queue.claim_next(claimant_id="worker-006", claim_request_id="m49-portable-claim-001")
|
||||
assert claim is not None
|
||||
running = queue.start(job.job_id, claim_token=claim.claim_token)
|
||||
return queue, running, claim.claim_token
|
||||
|
||||
|
||||
def _sealed_worker_job(expected: M49PortableSourceIdentity) -> SealedObservatoryRecordedJob:
|
||||
definition = _m49_definition()
|
||||
return SealedObservatoryRecordedJob(
|
||||
job_id=f"observatory-run-{'a' * 32}",
|
||||
request_sha256="2" * 64,
|
||||
identity_sha256="3" * 64,
|
||||
submission_receipt_sha256="6" * 64,
|
||||
source_session_id=SESSION_ID,
|
||||
source_catalog_sha256=expected.source_catalog_sha256,
|
||||
source_bundle_sha256=expected.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=(expected.source_capability_manifest_sha256),
|
||||
source_adapter_id=definition.source_adapter.adapter_id,
|
||||
source_adapter_version=definition.source_adapter.version,
|
||||
source_adapter_sha256=expected.source_adapter_sha256,
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
executor_release_id="m49-tgs-portable-executor-v1",
|
||||
executor_identity=ObservatoryWorkerExecutorIdentity(
|
||||
release_sha256="4" * 64,
|
||||
image_sha256="5" * 64,
|
||||
model_manifest_sha256=definition.model_manifest_sha256,
|
||||
resource_profile_sha256=definition.resource_profile.profile_sha256,
|
||||
),
|
||||
model_release_ids=(),
|
||||
resource_profile_id=definition.resource_profile.profile_id,
|
||||
checkpoint_policy=definition.resource_profile.checkpoint_policy,
|
||||
allowed_checkpoints=definition.resource_profile.allowed_checkpoints,
|
||||
claim_generation=1,
|
||||
claim_claimed_at_utc=NOW,
|
||||
claim_expires_at_utc="2026-08-31T09:05:00.000Z",
|
||||
claim_heartbeat_at_utc=NOW,
|
||||
claim_renewal_count=0,
|
||||
restart_from_zero=False,
|
||||
)
|
||||
|
||||
|
||||
def _seal_running_job(job: ObservatoryRecordedJob) -> SealedObservatoryRecordedJob:
|
||||
return SealedObservatoryRecordedJob(
|
||||
job_id=job.job_id,
|
||||
request_sha256=job.request_sha256,
|
||||
identity_sha256=job.identity_sha256,
|
||||
submission_receipt_sha256=job.submission_receipt_sha256,
|
||||
source_session_id=job.source_session_id,
|
||||
source_catalog_sha256=job.source_catalog_sha256,
|
||||
source_bundle_sha256=job.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
|
||||
source_adapter_id=job.source_adapter_id,
|
||||
source_adapter_version=job.source_adapter_version,
|
||||
source_adapter_sha256=job.source_adapter_sha256,
|
||||
setup_id=job.setup_id,
|
||||
definition_id=job.definition_id,
|
||||
definition_version=job.definition_version,
|
||||
definition_sha256=job.definition_sha256,
|
||||
executor_release_id=job.executor_release_id,
|
||||
executor_identity=ObservatoryWorkerExecutorIdentity(
|
||||
release_sha256=job.executor_release_sha256,
|
||||
image_sha256=job.executor_image_sha256,
|
||||
model_manifest_sha256=job.model_manifest_sha256,
|
||||
resource_profile_sha256=job.resource_profile_sha256,
|
||||
),
|
||||
model_release_ids=job.model_release_ids,
|
||||
resource_profile_id=job.resource_profile_id,
|
||||
checkpoint_policy=job.checkpoint_policy,
|
||||
allowed_checkpoints=job.allowed_checkpoints,
|
||||
claim_generation=job.claim_generation,
|
||||
claim_claimed_at_utc=job.claimed_at_utc,
|
||||
claim_expires_at_utc=job.claim_expires_at_utc,
|
||||
claim_heartbeat_at_utc=job.claim_heartbeat_at_utc,
|
||||
claim_renewal_count=job.claim_renewal_count,
|
||||
restart_from_zero=job.restart_from_zero,
|
||||
)
|
||||
|
||||
|
||||
def _worker_source_member(
|
||||
job: SealedObservatoryRecordedJob,
|
||||
*,
|
||||
kind: str,
|
||||
payload: bytes,
|
||||
media_type: str,
|
||||
artifact_id: str | None = None,
|
||||
primary: bool = False,
|
||||
camera_epoch: int | None = None,
|
||||
camera_sequence: int | None = None,
|
||||
) -> dict[str, object]:
|
||||
sha256 = hashlib.sha256(payload).hexdigest()
|
||||
identity = {
|
||||
"job_identity_sha256": job.identity_sha256,
|
||||
"source_bundle_sha256": job.source_bundle_sha256,
|
||||
"kind": kind,
|
||||
"artifact_id": artifact_id,
|
||||
"primary": primary,
|
||||
"camera_epoch": camera_epoch,
|
||||
"camera_sequence": camera_sequence,
|
||||
"media_type": media_type,
|
||||
"byte_length": len(payload),
|
||||
"sha256": sha256,
|
||||
}
|
||||
return {
|
||||
"member_id": hashlib.sha256(canonical_json(identity)).hexdigest(),
|
||||
"kind": kind,
|
||||
"media_type": media_type,
|
||||
"byte_length": len(payload),
|
||||
"sha256": sha256,
|
||||
"artifact_id": artifact_id,
|
||||
"primary": primary,
|
||||
"camera_epoch": camera_epoch,
|
||||
"camera_sequence": camera_sequence,
|
||||
}
|
||||
|
||||
|
||||
def _runner_outputs(stage_root: Path, tmp_path: Path) -> tuple[Path, Path]:
|
||||
stage = validate_m49_portable_source_stage(stage_root)
|
||||
rows = read_m49_source_index(
|
||||
stage.root / "sequence-index.ndjson",
|
||||
expected_frame_count=stage.timeline_frame_count,
|
||||
)
|
||||
output_root = tmp_path / "runner-outputs"
|
||||
output_root.mkdir()
|
||||
timing = [
|
||||
"timeline_frame_index\tsource_frame_index\tsession_seconds\tsample_available"
|
||||
"\tavailable_slot\tinput_points\tground_points\tnonground_points\ttgs_ms"
|
||||
"\tstage_wall_ms"
|
||||
]
|
||||
for row in rows:
|
||||
index = cast(int, row["timeline_frame_index"])
|
||||
if row["sample_available"] is not True:
|
||||
timing.append(
|
||||
f"{index}\t{row['source_frame_index']}\t"
|
||||
f"{cast(float, row['session_seconds']):.6f}"
|
||||
"\t0\t-1\t0\t0\t0\t0.000000\t0.010000"
|
||||
)
|
||||
continue
|
||||
source = np.fromfile(stage.root / cast(str, row["relative_path"]), dtype="<f4").reshape(
|
||||
-1, 4
|
||||
)
|
||||
ranges = np.linalg.norm(source[:, :2].astype(np.float64), axis=1)
|
||||
eligible = source[(ranges > 1.0) & (ranges < 80.0)]
|
||||
ground = eligible[:1]
|
||||
nonground = eligible[1:2]
|
||||
(output_root / f"{index}_ground.bin").write_bytes(ground.tobytes())
|
||||
(output_root / f"{index}_nonground.bin").write_bytes(nonground.tobytes())
|
||||
timing.append(
|
||||
f"{index}\t{row['source_frame_index']}\t"
|
||||
f"{cast(float, row['session_seconds']):.6f}"
|
||||
f"\t1\t{row['available_slot']}\t{source.shape[0]}\t{ground.shape[0]}"
|
||||
f"\t{nonground.shape[0]}\t1.250000\t1.500000"
|
||||
)
|
||||
timing_path = tmp_path / "tgs-timing.tsv"
|
||||
timing_path.write_text("\n".join(timing) + "\n", encoding="utf-8")
|
||||
return output_root, timing_path
|
||||
|
||||
|
||||
def test_dynamic_source_materializer_is_source_derived_and_tamper_evident(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
stage_root, _expected = _materialized_source(tmp_path, monkeypatch)
|
||||
stage = validate_m49_portable_source_stage(stage_root)
|
||||
assert stage.timeline_frame_count == 3
|
||||
assert stage.available_lidar_frame_count == 2
|
||||
rows = read_m49_source_index(stage.root / "sequence-index.ndjson", expected_frame_count=3)
|
||||
assert [row["sample_available"] for row in rows] == [False, True, True]
|
||||
assert [row["source_frame_index"] for row in rows] == [0, 1, 2]
|
||||
assert all(
|
||||
"RAVNOVES" not in path.read_text(errors="ignore")
|
||||
for path in (
|
||||
stage.root / "manifest.json",
|
||||
stage.root / "sequence-index.ndjson",
|
||||
stage.root / "schedule.tsv",
|
||||
)
|
||||
)
|
||||
|
||||
schedule = stage.root / "schedule.tsv"
|
||||
schedule.write_text(schedule.read_text() + "0\t0\t0\t-1\t0\n", encoding="utf-8")
|
||||
with pytest.raises(M49PortableSourceError):
|
||||
validate_m49_portable_source_stage(stage.root)
|
||||
|
||||
|
||||
def test_source_materializer_rejects_unadmitted_adjacent_metadata(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
with pytest.raises(M49PortableSourceError, match="admitted raw and host-time"):
|
||||
_materialized_source(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
include_metadata_member=False,
|
||||
)
|
||||
|
||||
|
||||
def test_fixed_worker_stage_consumer_requires_manifested_metadata_member(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
direct_root = tmp_path / "direct"
|
||||
direct_root.mkdir()
|
||||
_direct_stage, expected = _materialized_source(direct_root, monkeypatch)
|
||||
job = _sealed_worker_job(expected)
|
||||
worker_root = tmp_path / "worker-source"
|
||||
(worker_root / "camera" / "epoch-1" / "segments").mkdir(parents=True)
|
||||
bundle = (direct_root / "source-bundle.json").read_bytes()
|
||||
capability = (direct_root / "source-capability.json").read_bytes()
|
||||
init = b"exact-init"
|
||||
segments = (b"segment-1", b"segment-2", b"segment-3")
|
||||
(worker_root / "source-bundle.json").write_bytes(bundle)
|
||||
(worker_root / "source-capability.json").write_bytes(capability)
|
||||
(worker_root / "mqtt.raw.k1mqtt").write_bytes(RAW_PAYLOAD)
|
||||
(worker_root / "mqtt.metadata.jsonl").write_bytes(METADATA_PAYLOAD)
|
||||
(worker_root / "camera" / "epoch-1" / "init.mp4").write_bytes(init)
|
||||
for index, payload in enumerate(segments, start=1):
|
||||
(worker_root / "camera" / "epoch-1" / "segments" / f"{index}.m4s").write_bytes(payload)
|
||||
members = [
|
||||
_worker_source_member(
|
||||
job,
|
||||
kind="source-bundle",
|
||||
payload=bundle,
|
||||
media_type="application/json",
|
||||
),
|
||||
_worker_source_member(
|
||||
job,
|
||||
kind="source-capability",
|
||||
payload=capability,
|
||||
media_type="application/json",
|
||||
),
|
||||
_worker_source_member(
|
||||
job,
|
||||
kind="spatial-replay",
|
||||
payload=RAW_PAYLOAD,
|
||||
media_type="application/x-nodedc-k1mqtt",
|
||||
artifact_id="raw-transport-primary",
|
||||
primary=True,
|
||||
),
|
||||
_worker_source_member(
|
||||
job,
|
||||
kind="spatial-replay-metadata",
|
||||
payload=METADATA_PAYLOAD,
|
||||
media_type="application/x-ndjson",
|
||||
artifact_id="raw-transport-index",
|
||||
),
|
||||
_worker_source_member(
|
||||
job,
|
||||
kind="camera-init",
|
||||
payload=init,
|
||||
media_type="video/mp4",
|
||||
artifact_id="camera-primary",
|
||||
camera_epoch=1,
|
||||
),
|
||||
*[
|
||||
_worker_source_member(
|
||||
job,
|
||||
kind="camera-segment",
|
||||
payload=payload,
|
||||
media_type="video/iso.segment",
|
||||
artifact_id="camera-primary",
|
||||
camera_epoch=1,
|
||||
camera_sequence=index,
|
||||
)
|
||||
for index, payload in enumerate(segments, start=1)
|
||||
],
|
||||
]
|
||||
members.sort(key=lambda row: cast(str, row["member_id"]))
|
||||
materialization = {
|
||||
"schema_version": "missioncore.observatory-portable-source-materialization/v1",
|
||||
"job_id": job.job_id,
|
||||
"job_identity_sha256": job.identity_sha256,
|
||||
"claim_generation": job.claim_generation,
|
||||
"source": {
|
||||
"session_id": job.source_session_id,
|
||||
"bundle_sha256": job.source_bundle_sha256,
|
||||
"capability_manifest_sha256": job.source_capability_manifest_sha256,
|
||||
},
|
||||
"members": members,
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
(worker_root / "materialization-manifest.json").write_bytes(canonical_json(materialization))
|
||||
stage = PortableWorkerSourceStage(
|
||||
root=worker_root,
|
||||
source_bundle_sha256=job.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
|
||||
source_adapter_sha256=job.source_adapter_sha256,
|
||||
)
|
||||
built_from: list[Path] = []
|
||||
|
||||
def fake_build(capture: Path, output: Path, *, session_id: str) -> Path:
|
||||
assert session_id == SESSION_ID
|
||||
assert capture == worker_root / "mqtt.raw.k1mqtt"
|
||||
assert output.name == "lidar-replay-packs"
|
||||
built_from.append(capture)
|
||||
return worker_root
|
||||
|
||||
monkeypatch.setattr(source_module, "build_lidar_replay_pack_v2", fake_build)
|
||||
|
||||
class _DeliveredSource:
|
||||
def materialize(self, requested: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage:
|
||||
assert requested == job
|
||||
return stage
|
||||
|
||||
materializer = M49PortableSourceMaterializerAdapter(
|
||||
upstream=_DeliveredSource(),
|
||||
profile_path=PROFILE_PATH,
|
||||
output_parent=tmp_path / "worker-output",
|
||||
)
|
||||
materialized = materializer.materialize(job)
|
||||
assert built_from == [worker_root / "mqtt.raw.k1mqtt"]
|
||||
assert isinstance(materialized, M49PortableBoundSourceStage)
|
||||
assert materialized.m49_stage.timeline_frame_count == 3
|
||||
assert materialized.m49_stage.available_lidar_frame_count == 2
|
||||
|
||||
|
||||
def test_result_v2_assembler_and_exact_validator_round_trip(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
stage_root, expected = _materialized_source(tmp_path, monkeypatch)
|
||||
registry = _ready_m49_registry(tmp_path)
|
||||
definition = registry.resolve_setup("m49-tgs-portable-v2")
|
||||
queue, running, claim_token = _running_job(tmp_path, definition, expected)
|
||||
sealed = _seal_running_job(running)
|
||||
prepared = tmp_path / "prepared-runner"
|
||||
prepared.mkdir()
|
||||
output_root, timing_path = _runner_outputs(stage_root, prepared)
|
||||
binary = tmp_path / "run_m49_tgs_portable"
|
||||
binary.write_bytes(b"\x7fELFtest-only")
|
||||
binary.chmod(0o755)
|
||||
build_seal = builder.seal_m49_compiled_runner_build(
|
||||
source_root=REPOSITORY_ROOT,
|
||||
source_revision="f" * 40,
|
||||
binary_path=binary,
|
||||
manifest_path=tmp_path / "runner-build-seal.json",
|
||||
)
|
||||
installation = M49PortableRunnerInstallation(
|
||||
profile_path=PROFILE_PATH,
|
||||
runner_binary_path=binary,
|
||||
runner_build_seal_path=build_seal.manifest_path,
|
||||
runner_build_seal_sha256=build_seal.manifest_sha256,
|
||||
output_parent=tmp_path / "executor-output",
|
||||
)
|
||||
|
||||
def fake_invoke(
|
||||
*,
|
||||
binary: Path,
|
||||
sequence: Path,
|
||||
schedule: Path,
|
||||
output: Path,
|
||||
timing: Path,
|
||||
workspace: Path,
|
||||
timeout_seconds: int,
|
||||
) -> None:
|
||||
assert binary == installation.runner_binary_path
|
||||
assert sequence == Path(stage_root) / "tgs/sequence/velodyne"
|
||||
assert schedule == Path(stage_root) / "schedule.tsv"
|
||||
assert workspace.parent == installation.output_parent
|
||||
assert timeout_seconds == installation.timeout_seconds
|
||||
shutil.copytree(output_root, output)
|
||||
shutil.copyfile(timing_path, timing)
|
||||
|
||||
source_stage = validate_m49_portable_source_stage(stage_root)
|
||||
bound_source = M49PortableBoundSourceStage(
|
||||
root=source_stage.root,
|
||||
source_bundle_sha256=sealed.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=sealed.source_capability_manifest_sha256,
|
||||
source_adapter_sha256=sealed.source_adapter_sha256,
|
||||
job=sealed,
|
||||
m49_stage=source_stage,
|
||||
)
|
||||
runner = M49PortableProfileRunnerAdapter(
|
||||
definition=definition,
|
||||
installation=installation,
|
||||
created_at_utc=lambda: NOW,
|
||||
invoker=fake_invoke,
|
||||
)
|
||||
draft = runner.run(
|
||||
PortableWorkerRuntimePlan(
|
||||
job_id=sealed.job_id,
|
||||
adapter_id="m49-tgs-worker006-portable-v2",
|
||||
candidate_sha256="f" * 64,
|
||||
setup_id=sealed.setup_id,
|
||||
definition_sha256=sealed.definition_sha256,
|
||||
source_bundle_sha256=sealed.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=sealed.source_capability_manifest_sha256,
|
||||
result_contract_sha256=definition.result_contract.contract_sha256,
|
||||
phases=M49_PORTABLE_RUNTIME_PHASES,
|
||||
),
|
||||
bound_source,
|
||||
)
|
||||
package = PortableResultPackageManifest.from_bytes((draft.root / "manifest.json").read_bytes())
|
||||
assert draft.root.name == package.manifest_sha256
|
||||
assert draft.result_id.startswith("m49-tgs-portable-review-")
|
||||
succeeded = queue.succeed(
|
||||
running.job_id,
|
||||
claim_token=claim_token,
|
||||
result_id=draft.result_id,
|
||||
result_sha256=draft.result_sha256,
|
||||
)
|
||||
artifact_paths = {
|
||||
artifact.role: draft.root.joinpath(*Path(artifact.relative_path).parts)
|
||||
for artifact in package.artifacts
|
||||
}
|
||||
result_document = cast(
|
||||
dict[str, object],
|
||||
json.loads(artifact_paths["result-document"].read_bytes()),
|
||||
)
|
||||
context = PortableResultValidationContext(
|
||||
manifest=package,
|
||||
job=succeeded,
|
||||
definition=definition,
|
||||
result_document=result_document,
|
||||
artifact_paths=artifact_paths,
|
||||
)
|
||||
validate_m49_portable_result(context)
|
||||
|
||||
states = np.load(artifact_paths["costmap-states"], mmap_mode="r", allow_pickle=False)
|
||||
assert np.all(states[0] == 0)
|
||||
assert set(np.unique(states[1])).issubset({0, 1, 2, 3})
|
||||
assert 3 in states[1]
|
||||
point_accounting = cast(dict[str, object], result_document["point_accounting"])
|
||||
assert point_accounting["unaccounted"] == 0
|
||||
|
||||
drifted = copy.deepcopy(result_document)
|
||||
cast(dict[str, object], drifted["point_accounting"])["eligible"] = 999
|
||||
with pytest.raises(M49PortableResultError):
|
||||
validate_m49_portable_result(
|
||||
PortableResultValidationContext(
|
||||
manifest=package,
|
||||
job=succeeded,
|
||||
definition=definition,
|
||||
result_document=drifted,
|
||||
artifact_paths=artifact_paths,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_executor_release_candidate_is_deterministic_blocked_and_tamper_evident(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source_root = tmp_path / "source"
|
||||
for relative in builder.M49_RELEASE_SOURCES:
|
||||
target = source_root / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(REPOSITORY_ROOT / relative, target)
|
||||
first = builder.build_m49_executor_release_candidate(
|
||||
source_root=source_root,
|
||||
output_directory=tmp_path / "out-a",
|
||||
source_revision="f" * 40,
|
||||
source_state="uncommitted-candidate",
|
||||
)
|
||||
second = builder.build_m49_executor_release_candidate(
|
||||
source_root=source_root,
|
||||
output_directory=tmp_path / "out-b",
|
||||
source_revision="f" * 40,
|
||||
source_state="uncommitted-candidate",
|
||||
)
|
||||
assert first.archive.read_bytes() == second.archive.read_bytes()
|
||||
assert first.archive_sha256 == second.archive_sha256
|
||||
assert first.manifest["state"] == "blocked"
|
||||
assert tuple(first.manifest["blockers"]) == builder.M49_RELEASE_BLOCKERS
|
||||
assert first.manifest["executor_image_sha256"] is None
|
||||
assert first.manifest["compiled_runner"] is None
|
||||
assert builder.verify_m49_executor_release_candidate(first.archive) == first
|
||||
|
||||
dockerfile = DOCKERFILE_PATH.read_text(encoding="utf-8")
|
||||
assert "FROM ndc/mission-core-m49-t3-travel:20260826" in dockerfile
|
||||
assert builder.M49_TRAVEL_IMAGE_SHA256 in dockerfile
|
||||
assert "--network" not in dockerfile
|
||||
assert "com.nodedc.authority=\"observation-only\"" in dockerfile
|
||||
installer = INSTALLER_PATH.read_text(encoding="utf-8")
|
||||
assert "--pull=false --no-cache --network none" in installer
|
||||
assert "--network none --read-only" in installer
|
||||
assert "committed-source-snapshot-missing" in installer
|
||||
assert "MissionCore-M49TgsFullShadow" in installer
|
||||
|
||||
changed = source_root / "src" / "k1link" / "observatory" / "m49_portable_result.py"
|
||||
changed.write_bytes(changed.read_bytes() + b"\n")
|
||||
drifted = builder.build_m49_executor_release_candidate(
|
||||
source_root=source_root,
|
||||
output_directory=tmp_path / "out-c",
|
||||
source_revision="f" * 40,
|
||||
source_state="uncommitted-candidate",
|
||||
)
|
||||
assert drifted.candidate_sha256 != first.candidate_sha256
|
||||
@@ -0,0 +1,600 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
PortableArtifactTransportIntegrityError,
|
||||
PortableArtifactTransportUnavailableError,
|
||||
PortableObservatoryArtifactTransport,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
RESULT_DOCUMENT_ROLE,
|
||||
PortableResultArtifact,
|
||||
PortableResultPackageManifest,
|
||||
canonical_json,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJob,
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueStaleClaimError,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.source_admission import (
|
||||
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
|
||||
)
|
||||
from k1link.sessions.media import (
|
||||
RecordedMediaEpoch,
|
||||
RecordedMediaManifest,
|
||||
RecordedMediaSegment,
|
||||
)
|
||||
from k1link.sessions.models import (
|
||||
RecordedMediaArtifact,
|
||||
ReplayArtifact,
|
||||
ReplayCommand,
|
||||
SessionDetail,
|
||||
SessionSummary,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
NOW = "2026-08-31T10:00:00.000Z"
|
||||
SOURCE_SESSION_ID = "20260831T095500Z_viewer_live"
|
||||
RESULT_ID = "portable-result-transport-001"
|
||||
|
||||
|
||||
def _ready_registry(tmp_path: Path) -> PortableRunDefinitionRegistry:
|
||||
registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
base = registry.definitions[0]
|
||||
document = cast(
|
||||
dict[str, object], json.loads(REGISTRY_PATH.read_text(encoding="utf-8"))
|
||||
)
|
||||
rows = cast(list[object], document["definitions"])
|
||||
selected = copy.deepcopy(cast(dict[str, object], rows[0]))
|
||||
selected["executor"] = {
|
||||
"contour_id": "worker-006",
|
||||
"state": "ready",
|
||||
"release_id": "portable-transport-test-executor",
|
||||
"release_sha256": "1" * 64,
|
||||
"image_sha256": "2" * 64,
|
||||
"reason_code": None,
|
||||
"reason": None,
|
||||
}
|
||||
identity = copy.deepcopy(base.identity_document())
|
||||
identity["executor"] = {
|
||||
"contour_id": "worker-006",
|
||||
"state": "ready",
|
||||
"release_id": "portable-transport-test-executor",
|
||||
"release_sha256": "1" * 64,
|
||||
"image_sha256": "2" * 64,
|
||||
}
|
||||
selected["definition_sha256"] = canonical_sha256(identity)
|
||||
path = tmp_path / "definitions.json"
|
||||
path.write_bytes(
|
||||
canonical_json(
|
||||
{
|
||||
"schema_version": document["schema_version"],
|
||||
"definitions": [selected],
|
||||
}
|
||||
)
|
||||
)
|
||||
return PortableRunDefinitionRegistry.from_file(path)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Store:
|
||||
data_dir: Path
|
||||
detail: SessionDetail
|
||||
catalog_sha256: str
|
||||
replay: ReplayCommand
|
||||
recorded_media: tuple[RecordedMediaArtifact, ...]
|
||||
|
||||
def get_session_with_catalog_snapshot(
|
||||
self, session_id: str
|
||||
) -> tuple[SessionDetail, str]:
|
||||
assert session_id == SOURCE_SESSION_ID
|
||||
return self.detail, self.catalog_sha256
|
||||
|
||||
def prepare_replay(self, session_id: str) -> ReplayCommand:
|
||||
assert session_id == SOURCE_SESSION_ID
|
||||
return self.replay
|
||||
|
||||
def list_recorded_media(
|
||||
self, session_id: str
|
||||
) -> tuple[RecordedMediaArtifact, ...]:
|
||||
assert session_id == SOURCE_SESSION_ID
|
||||
return self.recorded_media
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Inspector:
|
||||
manifest: RecordedMediaManifest
|
||||
|
||||
def restore_prepared(
|
||||
self,
|
||||
artifact: RecordedMediaArtifact,
|
||||
replay: ReplayCommand,
|
||||
) -> RecordedMediaManifest:
|
||||
assert artifact.artifact_id == self.manifest.artifact_id
|
||||
assert replay.session_id == self.manifest.session_id
|
||||
return self.manifest
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Fixture:
|
||||
service: PortableObservatoryArtifactTransport
|
||||
queue: ObservatoryRecordedJobQueue
|
||||
definition: PortableRunDefinition
|
||||
job: ObservatoryRecordedJob
|
||||
claim_token: str
|
||||
raw_path: Path
|
||||
|
||||
|
||||
def _fixture(tmp_path: Path) -> _Fixture:
|
||||
registry = _ready_registry(tmp_path)
|
||||
definition = registry.definitions[0]
|
||||
data_dir = tmp_path / "data"
|
||||
source_root = tmp_path / "source"
|
||||
raw_path = source_root / "mqtt.raw.k1mqtt"
|
||||
metadata_path = source_root / "mqtt.metadata.jsonl"
|
||||
init_path = source_root / "camera" / "epoch-1" / "init.mp4"
|
||||
segment_path = source_root / "camera" / "epoch-1" / "segments" / "1.m4s"
|
||||
segment_path.parent.mkdir(parents=True)
|
||||
raw_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
raw_path.write_bytes(b"sealed-raw-replay")
|
||||
metadata_path.write_bytes(b'{"offset":0,"topic":"/points"}\n')
|
||||
init_path.write_bytes(b"sealed-init")
|
||||
segment_path.write_bytes(b"sealed-segment")
|
||||
raw_sha = hashlib.sha256(raw_path.read_bytes()).hexdigest()
|
||||
metadata_sha = hashlib.sha256(metadata_path.read_bytes()).hexdigest()
|
||||
init_sha = hashlib.sha256(init_path.read_bytes()).hexdigest()
|
||||
segment_sha = hashlib.sha256(segment_path.read_bytes()).hexdigest()
|
||||
catalog_sha = "3" * 64
|
||||
generation_sha = "4" * 64
|
||||
replay = ReplayCommand(
|
||||
session_id=SOURCE_SESSION_ID,
|
||||
plugin_id=definition.source_requirements.plugin_id,
|
||||
allowed_root=source_root,
|
||||
session_root=source_root,
|
||||
primary_artifact_id="raw-transport-primary",
|
||||
artifacts=(
|
||||
ReplayArtifact(
|
||||
artifact_id="raw-transport-primary",
|
||||
path=raw_path,
|
||||
media_type="application/x-nodedc-k1mqtt",
|
||||
file_byte_length=raw_path.stat().st_size,
|
||||
replay_byte_length=raw_path.stat().st_size,
|
||||
expected_sha256=raw_sha,
|
||||
),
|
||||
ReplayArtifact(
|
||||
artifact_id="raw-transport-index",
|
||||
path=metadata_path,
|
||||
media_type="application/x-ndjson",
|
||||
file_byte_length=metadata_path.stat().st_size,
|
||||
replay_byte_length=metadata_path.stat().st_size,
|
||||
expected_sha256=None,
|
||||
),
|
||||
),
|
||||
timeline_origin_epoch_ns=1,
|
||||
timeline_origin_monotonic_ns=2,
|
||||
speed=1.0,
|
||||
loop=False,
|
||||
)
|
||||
recorded_media = RecordedMediaArtifact(
|
||||
session_id=SOURCE_SESSION_ID,
|
||||
public_source_id="recorded.camera.right",
|
||||
artifact_id="recorded-video-right",
|
||||
source_path=source_root / "camera",
|
||||
byte_length=init_path.stat().st_size + segment_path.stat().st_size,
|
||||
)
|
||||
epoch = RecordedMediaEpoch(
|
||||
ordinal=1,
|
||||
path=init_path.parent,
|
||||
init_path=init_path,
|
||||
init_byte_length=init_path.stat().st_size,
|
||||
init_sha256=init_sha,
|
||||
media_type='video/mp4; codecs="avc1.641028"',
|
||||
timeline_start_seconds=0.0,
|
||||
timeline_end_seconds=0.1,
|
||||
segments=(
|
||||
RecordedMediaSegment(
|
||||
sequence=1,
|
||||
path=segment_path,
|
||||
byte_length=segment_path.stat().st_size,
|
||||
sha256=segment_sha,
|
||||
random_access=True,
|
||||
end_time_seconds=0.1,
|
||||
),
|
||||
),
|
||||
)
|
||||
manifest = RecordedMediaManifest(
|
||||
session_id=SOURCE_SESSION_ID,
|
||||
public_source_id=recorded_media.public_source_id,
|
||||
artifact_id=recorded_media.artifact_id,
|
||||
synchronization="host-arrival-best-effort",
|
||||
generation_sha256=generation_sha,
|
||||
timeline_start_seconds=0.0,
|
||||
timeline_end_seconds=0.1,
|
||||
byte_length=recorded_media.byte_length,
|
||||
epochs=(epoch,),
|
||||
)
|
||||
detail = SessionDetail(
|
||||
summary=SessionSummary(
|
||||
session_id=SOURCE_SESSION_ID,
|
||||
display_name="Portable source",
|
||||
status="ready",
|
||||
started_at_utc=NOW,
|
||||
completed_at_utc=NOW,
|
||||
duration_seconds=0.1,
|
||||
modalities=("point-cloud", "trajectory", "video"),
|
||||
source_count=3,
|
||||
total_bytes=100,
|
||||
replayable=True,
|
||||
origin=definition.source_requirements.archive_id,
|
||||
),
|
||||
sources=(),
|
||||
artifacts=(),
|
||||
plugin_id=definition.source_requirements.plugin_id,
|
||||
archive_id=definition.source_requirements.archive_id,
|
||||
)
|
||||
source_adapter = {
|
||||
"id": definition.source_adapter.adapter_id,
|
||||
"version": definition.source_adapter.version,
|
||||
"sha256": definition.source_adapter.contract_sha256,
|
||||
}
|
||||
bundle = {
|
||||
"schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
"source_session_id": SOURCE_SESSION_ID,
|
||||
"source_catalog_sha256": catalog_sha,
|
||||
"plugin_id": definition.source_requirements.plugin_id,
|
||||
"archive_id": definition.source_requirements.archive_id,
|
||||
"source_adapter": source_adapter,
|
||||
"sources": [],
|
||||
"spatial_replay": {
|
||||
"primary_artifact_id": replay.primary_artifact_id,
|
||||
"members": [
|
||||
{
|
||||
"artifact_id": "raw-transport-primary",
|
||||
"media_type": "application/x-nodedc-k1mqtt",
|
||||
"byte_length": raw_path.stat().st_size,
|
||||
"replay_byte_length": raw_path.stat().st_size,
|
||||
"sha256": raw_sha,
|
||||
},
|
||||
{
|
||||
"artifact_id": "raw-transport-index",
|
||||
"media_type": "application/x-ndjson",
|
||||
"byte_length": metadata_path.stat().st_size,
|
||||
"replay_byte_length": metadata_path.stat().st_size,
|
||||
"sha256": metadata_sha,
|
||||
},
|
||||
],
|
||||
"timeline_origin_epoch_ns": 1,
|
||||
"timeline_origin_monotonic_ns": 2,
|
||||
},
|
||||
"camera": {
|
||||
"artifact_id": recorded_media.artifact_id,
|
||||
"public_source_id": recorded_media.public_source_id,
|
||||
"generation_sha256": generation_sha,
|
||||
"synchronization": "host-arrival-best-effort",
|
||||
"epoch": {
|
||||
"ordinal": 1,
|
||||
"media_type": epoch.media_type,
|
||||
"init": {
|
||||
"byte_length": epoch.init_byte_length,
|
||||
"sha256": init_sha,
|
||||
},
|
||||
"timeline_start_seconds": 0.0,
|
||||
"timeline_end_seconds": 0.1,
|
||||
"segments": [
|
||||
{
|
||||
"sequence": 1,
|
||||
"byte_length": segment_path.stat().st_size,
|
||||
"sha256": segment_sha,
|
||||
"random_access": True,
|
||||
"end_time_seconds": 0.1,
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
"authority": OBSERVATION_ONLY_AUTHORITY,
|
||||
}
|
||||
bundle_payload = canonical_json(bundle)
|
||||
bundle_sha = hashlib.sha256(bundle_payload).hexdigest()
|
||||
capability = {
|
||||
"schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
"source_session_id": SOURCE_SESSION_ID,
|
||||
"source_catalog_sha256": catalog_sha,
|
||||
"source_bundle_sha256": bundle_sha,
|
||||
"source_adapter_sha256": definition.source_adapter.contract_sha256,
|
||||
"modalities": [],
|
||||
"camera_profile": {},
|
||||
"calibration": {},
|
||||
"authority": OBSERVATION_ONLY_AUTHORITY,
|
||||
}
|
||||
capability_payload = canonical_json(capability)
|
||||
capability_sha = hashlib.sha256(capability_payload).hexdigest()
|
||||
documents = data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||
documents.mkdir(parents=True)
|
||||
(documents / f"{bundle_sha}.json").write_bytes(bundle_payload)
|
||||
(documents / f"{capability_sha}.json").write_bytes(capability_payload)
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
data_dir,
|
||||
definitions=RecordedRunDefinitionRegistry(
|
||||
(definition.to_recorded_run_definition(),)
|
||||
),
|
||||
clock=lambda: NOW,
|
||||
)
|
||||
job, created = queue.submit(
|
||||
ObservatoryRecordedJobIntent(
|
||||
idempotency_key="portable-artifact-transport-001",
|
||||
source_session_id=SOURCE_SESSION_ID,
|
||||
source_catalog_sha256=catalog_sha,
|
||||
source_bundle_sha256=bundle_sha,
|
||||
source_capability_manifest_sha256=capability_sha,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
),
|
||||
enqueue=True,
|
||||
)
|
||||
assert created
|
||||
claim = queue.claim_next(
|
||||
claimant_id="worker-006",
|
||||
claim_request_id="portable-artifact-claim-001",
|
||||
)
|
||||
assert claim is not None
|
||||
running = queue.start(job.job_id, claim_token=claim.claim_token)
|
||||
store = _Store(
|
||||
data_dir=data_dir,
|
||||
detail=detail,
|
||||
catalog_sha256=catalog_sha,
|
||||
replay=replay,
|
||||
recorded_media=(recorded_media,),
|
||||
)
|
||||
service = PortableObservatoryArtifactTransport(
|
||||
queue=queue,
|
||||
session_store=store, # type: ignore[arg-type]
|
||||
media_inspector=_Inspector(manifest), # type: ignore[arg-type]
|
||||
definitions=registry,
|
||||
)
|
||||
return _Fixture(service, queue, definition, running, claim.claim_token, raw_path)
|
||||
|
||||
|
||||
def _result_package(
|
||||
tmp_path: Path,
|
||||
fixture: _Fixture,
|
||||
) -> tuple[PortableResultPackageManifest, bytes]:
|
||||
result_document = {
|
||||
"schema_version": fixture.definition.result_contract.result_schema,
|
||||
"result_id": RESULT_ID,
|
||||
"result_kind": fixture.definition.result_contract.result_kind,
|
||||
"authority": OBSERVATION_ONLY_AUTHORITY,
|
||||
}
|
||||
payload = canonical_json(result_document)
|
||||
artifact = PortableResultArtifact(
|
||||
role=RESULT_DOCUMENT_ROLE,
|
||||
relative_path="artifacts/result.json",
|
||||
media_type="application/json",
|
||||
byte_length=len(payload),
|
||||
sha256=hashlib.sha256(payload).hexdigest(),
|
||||
)
|
||||
package = PortableResultPackageManifest.create(
|
||||
job=fixture.job,
|
||||
definition=fixture.definition,
|
||||
result_id=RESULT_ID,
|
||||
created_at_utc=NOW,
|
||||
artifacts=(artifact,),
|
||||
)
|
||||
return package, payload
|
||||
|
||||
|
||||
def test_source_members_are_claim_bound_and_materialized_through_cas(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
fixture = _fixture(tmp_path)
|
||||
manifest = fixture.service.source_manifest(
|
||||
job_id=fixture.job.job_id,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
|
||||
assert {member.kind for member in manifest.members} == {
|
||||
"source-bundle",
|
||||
"source-capability",
|
||||
"spatial-replay",
|
||||
"spatial-replay-metadata",
|
||||
"camera-init",
|
||||
"camera-segment",
|
||||
}
|
||||
assert "path" not in json.dumps(manifest.as_dict(), sort_keys=True)
|
||||
raw = next(member for member in manifest.members if member.kind == "spatial-replay")
|
||||
admitted, cas_path = fixture.service.materialize_source_member(
|
||||
job_id=fixture.job.job_id,
|
||||
member_id=raw.member_id,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
|
||||
assert admitted.sha256 == raw.sha256
|
||||
assert cas_path.read_bytes() == b"sealed-raw-replay"
|
||||
assert cas_path != fixture.raw_path
|
||||
assert cas_path.name == raw.sha256
|
||||
metadata = next(
|
||||
member
|
||||
for member in manifest.members
|
||||
if member.kind == "spatial-replay-metadata"
|
||||
)
|
||||
admitted_metadata, metadata_cas_path = fixture.service.materialize_source_member(
|
||||
job_id=fixture.job.job_id,
|
||||
member_id=metadata.member_id,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
assert admitted_metadata.artifact_id == "raw-transport-index"
|
||||
assert metadata_cas_path.read_bytes() == b'{"offset":0,"topic":"/points"}\n'
|
||||
assert metadata_cas_path.name == metadata.sha256
|
||||
|
||||
|
||||
def test_source_member_rejects_tampering_and_stale_generation(tmp_path: Path) -> None:
|
||||
fixture = _fixture(tmp_path)
|
||||
manifest = fixture.service.source_manifest(
|
||||
job_id=fixture.job.job_id,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
raw = next(member for member in manifest.members if member.kind == "spatial-replay")
|
||||
fixture.raw_path.write_bytes(b"changed")
|
||||
|
||||
with pytest.raises(
|
||||
PortableArtifactTransportIntegrityError,
|
||||
match="admitted regular file|changed",
|
||||
):
|
||||
fixture.service.materialize_source_member(
|
||||
job_id=fixture.job.job_id,
|
||||
member_id=raw.member_id,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError):
|
||||
fixture.service.source_manifest(
|
||||
job_id=fixture.job.job_id,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation + 1,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
|
||||
|
||||
def test_result_upload_is_atomic_resumable_and_required_before_success(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
fixture = _fixture(tmp_path)
|
||||
package, result_payload = _result_package(tmp_path, fixture)
|
||||
plan = fixture.service.stage_result_manifest(
|
||||
job_id=fixture.job.job_id,
|
||||
result_sha256=package.manifest_sha256,
|
||||
manifest_payload=package.canonical_bytes,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
assert plan.complete is False
|
||||
with pytest.raises(PortableArtifactTransportUnavailableError, match="incomplete"):
|
||||
fixture.service.complete_result_upload(
|
||||
job_id=fixture.job.job_id,
|
||||
result_sha256=package.manifest_sha256,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
|
||||
async def chunks() -> object:
|
||||
yield result_payload[:7]
|
||||
yield result_payload[7:]
|
||||
|
||||
uploaded = asyncio.run(
|
||||
fixture.service.upload_result_member(
|
||||
job_id=fixture.job.job_id,
|
||||
result_sha256=package.manifest_sha256,
|
||||
member_id=plan.members[0].member_id,
|
||||
chunks=chunks(), # type: ignore[arg-type]
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
)
|
||||
assert uploaded.complete is True
|
||||
repeated = fixture.service.stage_result_manifest(
|
||||
job_id=fixture.job.job_id,
|
||||
result_sha256=package.manifest_sha256,
|
||||
manifest_payload=package.canonical_bytes,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
assert repeated.members[0].uploaded is True
|
||||
receipt = fixture.service.complete_result_upload(
|
||||
job_id=fixture.job.job_id,
|
||||
result_sha256=package.manifest_sha256,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
assert receipt.result_id == RESULT_ID
|
||||
package_root = fixture.service.require_completed_for_success(
|
||||
job_id=fixture.job.job_id,
|
||||
result_id=RESULT_ID,
|
||||
result_sha256=package.manifest_sha256,
|
||||
claim_token=fixture.claim_token,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
succeeded = fixture.queue.succeed(
|
||||
fixture.job.job_id,
|
||||
claim_token=fixture.claim_token,
|
||||
result_id=RESULT_ID,
|
||||
result_sha256=package.manifest_sha256,
|
||||
)
|
||||
assert fixture.service.package_root_for_terminal(succeeded) == package_root
|
||||
assert (package_root / "artifacts" / "result.json").read_bytes() == result_payload
|
||||
|
||||
|
||||
def test_result_upload_rejects_wrong_digest_and_never_publishes_partial_member(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
fixture = _fixture(tmp_path)
|
||||
package, result_payload = _result_package(tmp_path, fixture)
|
||||
plan = fixture.service.stage_result_manifest(
|
||||
job_id=fixture.job.job_id,
|
||||
result_sha256=package.manifest_sha256,
|
||||
manifest_payload=package.canonical_bytes,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
|
||||
async def bad_chunks() -> object:
|
||||
yield b"x" * len(result_payload)
|
||||
|
||||
with pytest.raises(
|
||||
PortableArtifactTransportIntegrityError,
|
||||
match="differs from its manifest",
|
||||
):
|
||||
asyncio.run(
|
||||
fixture.service.upload_result_member(
|
||||
job_id=fixture.job.job_id,
|
||||
result_sha256=package.manifest_sha256,
|
||||
member_id=plan.members[0].member_id,
|
||||
chunks=bad_chunks(), # type: ignore[arg-type]
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
)
|
||||
repeated = fixture.service.stage_result_manifest(
|
||||
job_id=fixture.job.job_id,
|
||||
result_sha256=package.manifest_sha256,
|
||||
manifest_payload=package.canonical_bytes,
|
||||
claim_token=fixture.claim_token,
|
||||
claim_generation=fixture.job.claim_generation,
|
||||
claimant_id="worker-006",
|
||||
)
|
||||
assert repeated.members[0].uploaded is False
|
||||
File diff suppressed because it is too large
Load Diff
@@ -367,6 +367,34 @@ def test_not_installed_definition_fails_before_source_or_queue_writes(
|
||||
assert not (tmp_path / "observatory-recorded-jobs.sqlite3").exists()
|
||||
|
||||
|
||||
def test_not_installed_model_free_definition_remains_capability_probeable(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = _blocked_registry()
|
||||
_write_probe_summary(tmp_path, segment_count=17)
|
||||
service, store, queue, inspector = _service(
|
||||
tmp_path,
|
||||
registry=registry,
|
||||
with_queue=False,
|
||||
)
|
||||
definition = registry.resolve_setup("m49-tgs-portable-v2")
|
||||
|
||||
capability = service.probe(
|
||||
source_session_id=SESSION_ID,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
)
|
||||
|
||||
assert capability.source_session_id == SESSION_ID
|
||||
assert capability.camera_segment_count == 17
|
||||
assert capability.source_adapter_sha256 == definition.source_adapter.contract_sha256
|
||||
assert store.catalog_reads == 1
|
||||
assert store.prepare_replay_calls == 0
|
||||
assert inspector.restore_calls == 0
|
||||
assert queue is None
|
||||
assert not (tmp_path / PORTABLE_SOURCE_DOCUMENT_DIRECTORY).exists()
|
||||
|
||||
|
||||
def test_probe_is_bounded_and_does_not_enter_replay_or_media_inspector(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
||||
@@ -0,0 +1,595 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.artifact_gateway import CentralArtifactStore
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||
RESULT_DOCUMENT_ROLE,
|
||||
PortableCalculationProfilePolicy,
|
||||
PortableCalculationProfileRegistry,
|
||||
PortableResultArtifact,
|
||||
PortableResultContractValidatorRegistration,
|
||||
PortableResultContractValidatorRegistry,
|
||||
PortableResultPackageIntegrityError,
|
||||
PortableResultPackageManifest,
|
||||
PortableResultPublicationBlockedError,
|
||||
PortableResultValidationContext,
|
||||
)
|
||||
from k1link.observatory.portable_result_publisher import (
|
||||
PortableObservatoryResultPublisher,
|
||||
resolve_published_portable_calculation_profile,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJob,
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.source_admission import (
|
||||
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
|
||||
)
|
||||
from k1link.sessions import (
|
||||
ObservationArchiveSource,
|
||||
ObservationSessionCandidate,
|
||||
SessionStore,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
NOW = "2026-08-31T08:00:00.000Z"
|
||||
SOURCE_SESSION_ID = "20260831T075500Z_viewer_live"
|
||||
RESULT_ID = "portable-lab-result-001"
|
||||
AUTHORITY = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _ready_registry(tmp_path: Path) -> PortableRunDefinitionRegistry:
|
||||
base_definition = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).definitions[0]
|
||||
document = cast(
|
||||
dict[str, object],
|
||||
json.loads(REGISTRY_PATH.read_text(encoding="utf-8")),
|
||||
)
|
||||
rows = cast(list[object], document["definitions"])
|
||||
selected = copy.deepcopy(cast(dict[str, object], rows[0]))
|
||||
selected["executor"] = {
|
||||
"contour_id": "worker-006",
|
||||
"state": "ready",
|
||||
"release_id": "lab-v1-portable-executor-v1",
|
||||
"release_sha256": "1" * 64,
|
||||
"image_sha256": "2" * 64,
|
||||
"reason_code": None,
|
||||
"reason": None,
|
||||
}
|
||||
identity = copy.deepcopy(base_definition.identity_document())
|
||||
identity["executor"] = {
|
||||
"contour_id": "worker-006",
|
||||
"state": "ready",
|
||||
"release_id": "lab-v1-portable-executor-v1",
|
||||
"release_sha256": "1" * 64,
|
||||
"image_sha256": "2" * 64,
|
||||
}
|
||||
selected["definition_sha256"] = canonical_sha256(identity)
|
||||
path = tmp_path / "portable-definitions.json"
|
||||
path.write_bytes(
|
||||
_canonical_json(
|
||||
{
|
||||
"schema_version": document["schema_version"],
|
||||
"definitions": [selected],
|
||||
}
|
||||
)
|
||||
)
|
||||
return PortableRunDefinitionRegistry.from_file(path)
|
||||
|
||||
|
||||
def _source_store(
|
||||
tmp_path: Path,
|
||||
definition: PortableRunDefinition,
|
||||
) -> tuple[SessionStore, str, str, str]:
|
||||
repository = tmp_path / "repository"
|
||||
archive_root = tmp_path / "source-archive"
|
||||
session_root = archive_root / SOURCE_SESSION_ID
|
||||
session_root.mkdir(parents=True)
|
||||
candidate = ObservationSessionCandidate(
|
||||
session_id=SOURCE_SESSION_ID,
|
||||
display_name="Portable result source",
|
||||
status="ready",
|
||||
started_at_utc="2026-08-31T07:55:00.000Z",
|
||||
completed_at_utc="2026-08-31T07:59:00.000Z",
|
||||
duration_seconds=240.0,
|
||||
modalities=(),
|
||||
replayable=False,
|
||||
total_bytes=0,
|
||||
allowed_root=archive_root,
|
||||
session_root=session_root,
|
||||
primary_replay_artifact_id=None,
|
||||
timeline_origin_epoch_ns=None,
|
||||
timeline_origin_monotonic_ns=None,
|
||||
sources=(),
|
||||
artifacts=(),
|
||||
)
|
||||
archive = ObservationArchiveSource(
|
||||
plugin_id=definition.source_requirements.plugin_id,
|
||||
archive_id=definition.source_requirements.archive_id,
|
||||
root=archive_root,
|
||||
discover=lambda _root: (candidate,),
|
||||
)
|
||||
store = SessionStore(repository, data_dir=tmp_path / "mission-core-data")
|
||||
assert store.reconcile_archive(archive) == (SOURCE_SESSION_ID,)
|
||||
_detail, catalog_sha256 = store.get_session_with_catalog_snapshot(SOURCE_SESSION_ID)
|
||||
|
||||
source_adapter = {
|
||||
"id": definition.source_adapter.adapter_id,
|
||||
"version": definition.source_adapter.version,
|
||||
"sha256": definition.source_adapter.contract_sha256,
|
||||
}
|
||||
bundle = {
|
||||
"schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
"source_session_id": SOURCE_SESSION_ID,
|
||||
"source_catalog_sha256": catalog_sha256,
|
||||
"plugin_id": definition.source_requirements.plugin_id,
|
||||
"archive_id": definition.source_requirements.archive_id,
|
||||
"source_adapter": source_adapter,
|
||||
"sources": [],
|
||||
"spatial_replay": {},
|
||||
"camera": {},
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
bundle_bytes = _canonical_json(bundle)
|
||||
bundle_sha256 = hashlib.sha256(bundle_bytes).hexdigest()
|
||||
capability = {
|
||||
"schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
"source_session_id": SOURCE_SESSION_ID,
|
||||
"source_catalog_sha256": catalog_sha256,
|
||||
"source_bundle_sha256": bundle_sha256,
|
||||
"source_adapter_sha256": definition.source_adapter.contract_sha256,
|
||||
"modalities": [],
|
||||
"camera_profile": {},
|
||||
"calibration": {},
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
capability_bytes = _canonical_json(capability)
|
||||
capability_sha256 = hashlib.sha256(capability_bytes).hexdigest()
|
||||
source_documents = store.data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||
source_documents.mkdir()
|
||||
(source_documents / f"{bundle_sha256}.json").write_bytes(bundle_bytes)
|
||||
(source_documents / f"{capability_sha256}.json").write_bytes(capability_bytes)
|
||||
return store, catalog_sha256, bundle_sha256, capability_sha256
|
||||
|
||||
|
||||
def _running_job(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
definition: PortableRunDefinition,
|
||||
catalog_sha256: str,
|
||||
bundle_sha256: str,
|
||||
capability_sha256: str,
|
||||
) -> tuple[ObservatoryRecordedJobQueue, ObservatoryRecordedJob, str]:
|
||||
recorded = definition.to_recorded_run_definition()
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path / "mission-core-data",
|
||||
definitions=RecordedRunDefinitionRegistry((recorded,)),
|
||||
clock=lambda: NOW,
|
||||
)
|
||||
job, created = queue.submit(
|
||||
ObservatoryRecordedJobIntent(
|
||||
idempotency_key="portable-result-publication-001",
|
||||
source_session_id=SOURCE_SESSION_ID,
|
||||
source_catalog_sha256=catalog_sha256,
|
||||
source_bundle_sha256=bundle_sha256,
|
||||
source_capability_manifest_sha256=capability_sha256,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
),
|
||||
enqueue=True,
|
||||
)
|
||||
assert created is True
|
||||
claim = queue.claim_next(
|
||||
claimant_id="worker-006",
|
||||
claim_request_id="portable-result-claim-001",
|
||||
)
|
||||
assert claim is not None
|
||||
running = queue.start(job.job_id, claim_token=claim.claim_token)
|
||||
assert running.state == "running"
|
||||
return queue, running, claim.claim_token
|
||||
|
||||
|
||||
def _package(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
job: ObservatoryRecordedJob,
|
||||
definition: PortableRunDefinition,
|
||||
result_id: str = RESULT_ID,
|
||||
accepted: bool = True,
|
||||
) -> tuple[Path, PortableResultPackageManifest]:
|
||||
result_document = {
|
||||
"schema_version": definition.result_contract.result_schema,
|
||||
"result_id": result_id,
|
||||
"result_kind": definition.result_contract.result_kind,
|
||||
"accepted": accepted,
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
result_bytes = _canonical_json(result_document)
|
||||
artifact = PortableResultArtifact(
|
||||
role=RESULT_DOCUMENT_ROLE,
|
||||
relative_path="artifacts/result.json",
|
||||
media_type="application/json",
|
||||
byte_length=len(result_bytes),
|
||||
sha256=hashlib.sha256(result_bytes).hexdigest(),
|
||||
)
|
||||
package = PortableResultPackageManifest.create(
|
||||
job=job,
|
||||
definition=definition,
|
||||
result_id=result_id,
|
||||
created_at_utc=NOW,
|
||||
artifacts=(artifact,),
|
||||
)
|
||||
root = tmp_path / "packages" / package.manifest_sha256
|
||||
(root / "artifacts").mkdir(parents=True)
|
||||
(root / "manifest.json").write_bytes(package.canonical_bytes)
|
||||
(root / "artifacts" / "result.json").write_bytes(result_bytes)
|
||||
return root, package
|
||||
|
||||
|
||||
def _profile(definition: PortableRunDefinition) -> PortableCalculationProfilePolicy:
|
||||
return PortableCalculationProfilePolicy(
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
lab_id="LAB V1",
|
||||
display_name="LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||
)
|
||||
|
||||
|
||||
def _validator(context: PortableResultValidationContext) -> None:
|
||||
expected = {
|
||||
"schema_version": context.definition.result_contract.result_schema,
|
||||
"result_id": context.job.result_id,
|
||||
"result_kind": context.definition.result_contract.result_kind,
|
||||
"accepted": True,
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
if dict(context.result_document) != expected:
|
||||
raise ValueError("result contract payload was not accepted")
|
||||
|
||||
|
||||
def _publisher(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
store: SessionStore,
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
profile: PortableCalculationProfilePolicy | None,
|
||||
validator: Callable[[PortableResultValidationContext], None] | None,
|
||||
) -> PortableObservatoryResultPublisher:
|
||||
definition = registry.definitions[0]
|
||||
registrations = (
|
||||
()
|
||||
if validator is None
|
||||
else (
|
||||
PortableResultContractValidatorRegistration(
|
||||
contract_sha256=definition.result_contract.contract_sha256,
|
||||
validator=validator,
|
||||
),
|
||||
)
|
||||
)
|
||||
return PortableObservatoryResultPublisher(
|
||||
session_store=store,
|
||||
artifact_store=CentralArtifactStore(tmp_path / "central-artifacts", create=True),
|
||||
definitions=registry,
|
||||
calculation_profiles=PortableCalculationProfileRegistry(
|
||||
() if profile is None else (profile,)
|
||||
),
|
||||
validators=PortableResultContractValidatorRegistry(registrations),
|
||||
)
|
||||
|
||||
|
||||
def _fixture(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
result_id: str = RESULT_ID,
|
||||
accepted: bool = True,
|
||||
) -> tuple[
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinition,
|
||||
SessionStore,
|
||||
ObservatoryRecordedJob,
|
||||
Path,
|
||||
]:
|
||||
registry = _ready_registry(tmp_path)
|
||||
definition = registry.definitions[0]
|
||||
store, catalog_sha, bundle_sha, capability_sha = _source_store(tmp_path, definition)
|
||||
queue, running, claim_token = _running_job(
|
||||
tmp_path,
|
||||
definition=definition,
|
||||
catalog_sha256=catalog_sha,
|
||||
bundle_sha256=bundle_sha,
|
||||
capability_sha256=capability_sha,
|
||||
)
|
||||
package_root, package = _package(
|
||||
tmp_path,
|
||||
job=running,
|
||||
definition=definition,
|
||||
result_id=result_id,
|
||||
accepted=accepted,
|
||||
)
|
||||
succeeded = queue.succeed(
|
||||
running.job_id,
|
||||
claim_token=claim_token,
|
||||
result_id=result_id,
|
||||
result_sha256=package.manifest_sha256,
|
||||
)
|
||||
return registry, definition, store, succeeded, package_root
|
||||
|
||||
|
||||
def test_verified_package_publishes_immutable_binding_and_profile_provenance(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry, definition, store, job, package_root = _fixture(tmp_path)
|
||||
publisher = _publisher(
|
||||
tmp_path,
|
||||
store=store,
|
||||
registry=registry,
|
||||
profile=_profile(definition),
|
||||
validator=_validator,
|
||||
)
|
||||
|
||||
first = publisher.publish(job=job, package_root=package_root)
|
||||
second = publisher.publish(job=job, package_root=package_root)
|
||||
|
||||
assert second.binding == first.binding
|
||||
assert second.artifact_manifest == first.artifact_manifest
|
||||
assert first.binding.session_id == RESULT_ID
|
||||
assert first.binding.source_session_id == SOURCE_SESSION_ID
|
||||
assert first.binding.config_sha256 == definition.definition_sha256
|
||||
assert first.binding.replay_capability is None
|
||||
assert first.binding.provenance["calculation_profile"] == {
|
||||
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||
"setup_id": definition.setup_id,
|
||||
"display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||
"origin": "archived-definition",
|
||||
"definition_id": definition.definition_id,
|
||||
"definition_version": definition.version,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
}
|
||||
package_provenance = cast(dict[str, object], first.binding.provenance["result_package"])
|
||||
assert package_provenance["manifest_sha256"] == job.result_sha256
|
||||
assert package_provenance["artifact_manifest_id"] == first.artifact_manifest.manifest_id
|
||||
assert store.get_lab_instance(RESULT_ID) == first.binding
|
||||
assert store.get_session(SOURCE_SESSION_ID).summary.lab is None
|
||||
|
||||
summary = store.get_session(RESULT_ID).summary
|
||||
assert summary.display_name == (
|
||||
"Portable result source · полный маршрут и воспроизведение"
|
||||
)
|
||||
profiles = PortableCalculationProfileRegistry((_profile(definition),))
|
||||
assert resolve_published_portable_calculation_profile(
|
||||
summary,
|
||||
definitions=registry,
|
||||
calculation_profiles=profiles,
|
||||
) == _profile(definition).as_dict()
|
||||
|
||||
assert summary.lab is not None
|
||||
drifted_provenance = copy.deepcopy(summary.lab.provenance)
|
||||
drifted_provenance["calculation_profile_sha256"] = "0" * 64
|
||||
drifted = replace(
|
||||
summary,
|
||||
lab=replace(summary.lab, provenance=drifted_provenance),
|
||||
)
|
||||
assert (
|
||||
resolve_published_portable_calculation_profile(
|
||||
drifted,
|
||||
definitions=registry,
|
||||
calculation_profiles=profiles,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_result_contract_fails_before_artifacts_or_catalog_are_published(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry, definition, store, job, package_root = _fixture(tmp_path)
|
||||
publisher = _publisher(
|
||||
tmp_path,
|
||||
store=store,
|
||||
registry=registry,
|
||||
profile=_profile(definition),
|
||||
validator=None,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
PortableResultPublicationBlockedError,
|
||||
match="validator is not installed",
|
||||
):
|
||||
publisher.publish(job=job, package_root=package_root)
|
||||
|
||||
assert store.get_lab_instance(RESULT_ID) is None
|
||||
assert not tuple((tmp_path / "central-artifacts").glob("manifests/sha256/*/*"))
|
||||
|
||||
|
||||
def test_missing_definition_bound_profile_is_not_inferred_from_result_or_ui(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry, _definition, store, job, package_root = _fixture(tmp_path)
|
||||
publisher = _publisher(
|
||||
tmp_path,
|
||||
store=store,
|
||||
registry=registry,
|
||||
profile=None,
|
||||
validator=_validator,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
PortableResultPublicationBlockedError,
|
||||
match="profile policy is not registered",
|
||||
):
|
||||
publisher.publish(job=job, package_root=package_root)
|
||||
|
||||
assert store.get_lab_instance(RESULT_ID) is None
|
||||
|
||||
|
||||
def test_changed_artifact_bytes_fail_closed_before_catalog_publication(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry, definition, store, job, package_root = _fixture(tmp_path)
|
||||
(package_root / "artifacts" / "result.json").write_bytes(b"x" * 8)
|
||||
publisher = _publisher(
|
||||
tmp_path,
|
||||
store=store,
|
||||
registry=registry,
|
||||
profile=_profile(definition),
|
||||
validator=_validator,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
PortableResultPackageIntegrityError,
|
||||
match="artifact content changed",
|
||||
):
|
||||
publisher.publish(job=job, package_root=package_root)
|
||||
|
||||
assert store.get_lab_instance(RESULT_ID) is None
|
||||
|
||||
|
||||
def test_contract_validator_rejection_never_becomes_a_lab_result(tmp_path: Path) -> None:
|
||||
registry, definition, store, job, package_root = _fixture(
|
||||
tmp_path,
|
||||
accepted=False,
|
||||
)
|
||||
publisher = _publisher(
|
||||
tmp_path,
|
||||
store=store,
|
||||
registry=registry,
|
||||
profile=_profile(definition),
|
||||
validator=_validator,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
PortableResultPackageIntegrityError,
|
||||
match="failed its exact contract validator",
|
||||
):
|
||||
publisher.publish(job=job, package_root=package_root)
|
||||
|
||||
assert store.get_lab_instance(RESULT_ID) is None
|
||||
|
||||
|
||||
def test_missing_persisted_source_contract_is_a_publication_blocker(tmp_path: Path) -> None:
|
||||
registry, definition, store, job, package_root = _fixture(tmp_path)
|
||||
source_documents = store.data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||
for path in source_documents.iterdir():
|
||||
path.unlink()
|
||||
publisher = _publisher(
|
||||
tmp_path,
|
||||
store=store,
|
||||
registry=registry,
|
||||
profile=_profile(definition),
|
||||
validator=_validator,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
PortableResultPublicationBlockedError,
|
||||
match="source document is unavailable",
|
||||
):
|
||||
publisher.publish(job=job, package_root=package_root)
|
||||
|
||||
assert store.get_lab_instance(RESULT_ID) is None
|
||||
|
||||
|
||||
def test_package_manifest_rejects_noncanonical_or_authority_elevating_documents(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
registry = _ready_registry(tmp_path)
|
||||
definition = registry.definitions[0]
|
||||
store, catalog_sha, bundle_sha, capability_sha = _source_store(tmp_path, definition)
|
||||
_queue, running, _claim_token = _running_job(
|
||||
tmp_path,
|
||||
definition=definition,
|
||||
catalog_sha256=catalog_sha,
|
||||
bundle_sha256=bundle_sha,
|
||||
capability_sha256=capability_sha,
|
||||
)
|
||||
_root, manifest = _package(
|
||||
tmp_path,
|
||||
job=running,
|
||||
definition=definition,
|
||||
)
|
||||
elevated = manifest.as_dict()
|
||||
authority = cast(dict[str, object], elevated["authority"])
|
||||
authority["commands_enabled"] = True
|
||||
identity = {
|
||||
"schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||
**{
|
||||
key: value
|
||||
for key, value in elevated.items()
|
||||
if key not in {"schema_version", "identity_sha256"}
|
||||
},
|
||||
}
|
||||
elevated["identity_sha256"] = canonical_sha256(identity)
|
||||
|
||||
with pytest.raises(
|
||||
PortableResultPackageIntegrityError,
|
||||
match="not observation-only",
|
||||
):
|
||||
PortableResultPackageManifest.from_bytes(_canonical_json(elevated))
|
||||
with pytest.raises(
|
||||
PortableResultPackageIntegrityError,
|
||||
match="not canonical JSON",
|
||||
):
|
||||
PortableResultPackageManifest.from_bytes(
|
||||
json.dumps(manifest.as_dict(), indent=2).encode()
|
||||
)
|
||||
|
||||
|
||||
def test_legacy_canonical_result_namespace_cannot_be_republished(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
legacy_result_id = "lab-v1-vegetation-shadow-" + "a" * 64
|
||||
registry, definition, store, job, package_root = _fixture(
|
||||
tmp_path,
|
||||
result_id=legacy_result_id,
|
||||
)
|
||||
publisher = _publisher(
|
||||
tmp_path,
|
||||
store=store,
|
||||
registry=registry,
|
||||
profile=_profile(definition),
|
||||
validator=_validator,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
PortableResultPublicationBlockedError,
|
||||
match="legacy canonical result namespace",
|
||||
):
|
||||
publisher.publish(job=job, package_root=package_root)
|
||||
|
||||
assert store.get_lab_instance(legacy_result_id) is None
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import FrozenInstanceError, replace
|
||||
from pathlib import Path
|
||||
@@ -18,8 +19,10 @@ from k1link.observatory.portable_run_definitions import (
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
DEFINITION_SHA256 = "57bf8f0859e10e54e30322c9a8aa28b427699f6fe6b5267e279ec3390fa78466"
|
||||
DEFINITION_SHA256 = "3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2"
|
||||
MODEL_MANIFEST_SHA256 = "3fd2d43af73bd73f89d9ffae95d8770cfdeb46033ec967509124fac6ae4afe56"
|
||||
M49_DEFINITION_SHA256 = "73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910"
|
||||
M49_MODEL_MANIFEST_SHA256 = "489a43448f720a9b5c7993dc8279d167b77191a586f0d87b6d38b81cf728e2f1"
|
||||
|
||||
|
||||
def _registry() -> PortableRunDefinitionRegistry:
|
||||
@@ -96,6 +99,54 @@ def test_source_requirements_map_exactly_to_admission_contract() -> None:
|
||||
assert admission.adapter_sha256 == definition.source_adapter.contract_sha256
|
||||
|
||||
|
||||
def test_m49_portable_v2_is_model_free_and_contains_no_exact_source_binding() -> None:
|
||||
definition = _registry().resolve_setup("m49-tgs-portable-v2")
|
||||
|
||||
assert definition.definition_sha256 == M49_DEFINITION_SHA256
|
||||
assert definition.models == ()
|
||||
assert definition.learned_models == ()
|
||||
assert definition.model_manifest_sha256 == M49_MODEL_MANIFEST_SHA256
|
||||
assert definition.resource_profile.accelerator_id == "cpu-only"
|
||||
assert definition.executor.state == "not-installed"
|
||||
assert definition.executor.release_id is None
|
||||
assert definition.executor.release_sha256 is None
|
||||
assert definition.executor.image_sha256 is None
|
||||
identity = json.dumps(definition.identity_document(), sort_keys=True)
|
||||
assert "RAVNOVES00" not in identity
|
||||
assert "20260720T065719Z_viewer_live" not in identity
|
||||
assert "4489" not in identity
|
||||
assert "3928" not in identity
|
||||
|
||||
profile_path = REPOSITORY_ROOT / "config" / "perception" / "m49-tgs-portable-v2.json"
|
||||
profile = json.loads(profile_path.read_text(encoding="utf-8"))
|
||||
components = {component.component_id: component for component in definition.components}
|
||||
assert hashlib.sha256(profile_path.read_bytes()).hexdigest() == (
|
||||
components["m49-tgs-portable-profile-v2"].sha256
|
||||
)
|
||||
assert profile["source_binding"] == {
|
||||
"mode": "admitted-k1-recording",
|
||||
"camera_timeline": "dynamic",
|
||||
"lidar_replay": "dynamic",
|
||||
"trajectory": "dynamic",
|
||||
"frame_counts": "source-derived",
|
||||
"filesystem_paths": "executor-resolved",
|
||||
}
|
||||
|
||||
with pytest.raises(PortableRunDefinitionRegistryError, match="runner"):
|
||||
replace(
|
||||
definition,
|
||||
executor=PortableExecutorAvailability(
|
||||
contour_id="worker-006",
|
||||
state="ready",
|
||||
release_id="m49-tgs-portable-executor-v2",
|
||||
release_sha256="1" * 64,
|
||||
image_sha256="2" * 64,
|
||||
reason_code=None,
|
||||
reason=None,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_all_canonical_identities_are_recomputed_from_typed_content() -> None:
|
||||
definition = _registry().definitions[0]
|
||||
|
||||
@@ -278,6 +329,33 @@ def test_conversion_to_recorded_definition_requires_and_preserves_sealed_identit
|
||||
assert recorded.checkpoint_policy == "non-checkpointable"
|
||||
|
||||
|
||||
def test_blocked_definition_does_not_hide_an_unrelated_ready_definition() -> None:
|
||||
registry = _registry()
|
||||
blocked_lab = registry.resolve_setup("lab-v1-eomt-ddrnet-portable-v1")
|
||||
blocked_m49 = registry.resolve_setup("m49-tgs-portable-v2")
|
||||
ready_executor = PortableExecutorAvailability(
|
||||
contour_id="worker-006",
|
||||
state="ready",
|
||||
release_id="lab-v1-eomt-ddrnet-executor-v1",
|
||||
release_sha256="1" * 64,
|
||||
image_sha256="2" * 64,
|
||||
reason_code=None,
|
||||
reason=None,
|
||||
)
|
||||
identity = blocked_lab.identity_document()
|
||||
identity["executor"] = ready_executor.identity_document()
|
||||
ready_lab = replace(
|
||||
blocked_lab,
|
||||
executor=ready_executor,
|
||||
definition_sha256=canonical_sha256(identity),
|
||||
)
|
||||
mixed = PortableRunDefinitionRegistry((ready_lab, blocked_m49))
|
||||
|
||||
assert mixed.ready_recorded_definitions() == (ready_lab.to_recorded_run_definition(),)
|
||||
assert mixed.to_recorded_registry().definitions == (ready_lab.to_recorded_run_definition(),)
|
||||
assert mixed.resolve_setup("m49-tgs-portable-v2") is blocked_m49
|
||||
|
||||
|
||||
def test_production_lab_v1_model_component_and_result_identities_are_exact() -> None:
|
||||
definition = _registry().definitions[0]
|
||||
models = {model.release_id: model for model in definition.models}
|
||||
|
||||
@@ -1,13 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
|
||||
from k1link.observatory.portable_setup_projection import PortableLabV1SetupProjector
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableExecutorAvailability,
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjector,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.source_admission import PortableRecordedSourceCapability
|
||||
from k1link.sessions import SessionNotFoundError
|
||||
from k1link.sessions.models import SessionSummary
|
||||
@@ -111,4 +124,193 @@ def test_portable_setup_catalog_preserves_source_and_optional_slice_failures() -
|
||||
params={"source_session_id": SOURCE_SESSION_ID},
|
||||
)
|
||||
assert unavailable.status_code == 503
|
||||
assert unavailable.json()["detail"] == "Portable-каталог LAB V1 недоступен."
|
||||
assert unavailable.json()["detail"] == "Portable-каталог сетапов недоступен."
|
||||
|
||||
|
||||
def _ready_lab_registry() -> PortableRunDefinitionRegistry:
|
||||
blocked = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH).resolve_setup(
|
||||
"lab-v1-eomt-ddrnet-portable-v1"
|
||||
)
|
||||
executor = PortableExecutorAvailability(
|
||||
contour_id="worker-006",
|
||||
state="ready",
|
||||
release_id="lab-v1-eomt-ddrnet-executor-v1",
|
||||
release_sha256="1" * 64,
|
||||
image_sha256="2" * 64,
|
||||
reason_code=None,
|
||||
reason=None,
|
||||
)
|
||||
identity = blocked.identity_document()
|
||||
identity["executor"] = executor.identity_document()
|
||||
return PortableRunDefinitionRegistry(
|
||||
(
|
||||
replace(
|
||||
blocked,
|
||||
executor=executor,
|
||||
definition_sha256=canonical_sha256(identity),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class _PortableBinding:
|
||||
def __init__(
|
||||
self,
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
) -> None:
|
||||
self.registry = registry
|
||||
self.queue = queue
|
||||
self.check_sha256 = "9" * 64
|
||||
self.check_count = 0
|
||||
self.submit_count = 0
|
||||
|
||||
def probe(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> PortableRecordedSourceCapability:
|
||||
definition = self.registry.resolve(setup_id, definition_sha256)
|
||||
return PortableRecordedSourceCapability(
|
||||
source_session_id=source_session_id,
|
||||
source_catalog_sha256="a" * 64,
|
||||
source_adapter_sha256=definition.source_adapter.contract_sha256,
|
||||
camera_segment_count=600,
|
||||
)
|
||||
|
||||
def check(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> SimpleNamespace:
|
||||
self.registry.resolve(setup_id, definition_sha256)
|
||||
assert source_session_id == SOURCE_SESSION_ID
|
||||
self.check_count += 1
|
||||
return SimpleNamespace(check_sha256=self.check_sha256)
|
||||
|
||||
def submit(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
expected_check_sha256: str,
|
||||
idempotency_key: str,
|
||||
) -> tuple[object, bool]:
|
||||
self.registry.resolve(setup_id, definition_sha256)
|
||||
assert source_session_id == SOURCE_SESSION_ID
|
||||
assert expected_check_sha256 == self.check_sha256
|
||||
self.submit_count += 1
|
||||
return self.queue.submit(
|
||||
ObservatoryRecordedJobIntent(
|
||||
idempotency_key=idempotency_key,
|
||||
source_session_id=source_session_id,
|
||||
source_catalog_sha256="a" * 64,
|
||||
source_bundle_sha256="b" * 64,
|
||||
source_capability_manifest_sha256="c" * 64,
|
||||
setup_id=setup_id,
|
||||
definition_sha256=definition_sha256,
|
||||
),
|
||||
enqueue=True,
|
||||
)
|
||||
|
||||
|
||||
def test_portable_api_check_sha_fences_ready_submission(tmp_path: Path) -> None:
|
||||
registry = _ready_lab_registry()
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=RecordedRunDefinitionRegistry(registry.ready_recorded_definitions()),
|
||||
)
|
||||
binding = _PortableBinding(registry, queue)
|
||||
projector = PortableSetupProjector(
|
||||
registry=registry,
|
||||
capability_probe=binding, # type: ignore[arg-type]
|
||||
dispatch_available=True,
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_observatory_router(
|
||||
_Store(), # type: ignore[arg-type]
|
||||
portable_setup_projector=projector,
|
||||
portable_binding_service=binding, # type: ignore[arg-type]
|
||||
recorded_job_queue=queue,
|
||||
)
|
||||
)
|
||||
client = TestClient(app)
|
||||
definition = registry.definitions[0]
|
||||
preflight = client.post(
|
||||
"/api/v1/observatory/run-preflights",
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-run-preflight-request/v1",
|
||||
"source_session_id": SOURCE_SESSION_ID,
|
||||
"setup_id": definition.setup_id,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
},
|
||||
)
|
||||
|
||||
assert preflight.status_code == 200
|
||||
assert preflight.json()["outcome"] == "queueable"
|
||||
assert preflight.json()["check_sha256"] == binding.check_sha256
|
||||
assert binding.check_count == 1
|
||||
|
||||
submitted = client.post(
|
||||
"/api/v1/observatory/runs",
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-recorded-run-submit/v1",
|
||||
"idempotency_key": "portable:lab-v1:source-005",
|
||||
"source_session_id": SOURCE_SESSION_ID,
|
||||
"setup_id": definition.setup_id,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"check_sha256": binding.check_sha256,
|
||||
},
|
||||
)
|
||||
|
||||
assert submitted.status_code == 202
|
||||
assert submitted.json()["state"] == "queued"
|
||||
assert submitted.json()["setup"]["setup_id"] == definition.setup_id
|
||||
assert binding.submit_count == 1
|
||||
|
||||
|
||||
def test_portable_api_rejects_blocked_executor_before_binding_submit(tmp_path: Path) -> None:
|
||||
full_registry = PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
ready_registry = _ready_lab_registry()
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=RecordedRunDefinitionRegistry(ready_registry.ready_recorded_definitions()),
|
||||
)
|
||||
binding = _PortableBinding(full_registry, queue)
|
||||
projector = PortableSetupProjector(
|
||||
registry=full_registry,
|
||||
capability_probe=binding, # type: ignore[arg-type]
|
||||
dispatch_available=True,
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_observatory_router(
|
||||
_Store(), # type: ignore[arg-type]
|
||||
portable_setup_projector=projector,
|
||||
portable_binding_service=binding, # type: ignore[arg-type]
|
||||
recorded_job_queue=queue,
|
||||
)
|
||||
)
|
||||
definition = full_registry.resolve_setup("m49-tgs-portable-v2")
|
||||
response = TestClient(app).post(
|
||||
"/api/v1/observatory/runs",
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-recorded-run-submit/v1",
|
||||
"idempotency_key": "portable:m49:blocked",
|
||||
"source_session_id": SOURCE_SESSION_ID,
|
||||
"setup_id": definition.setup_id,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"check_sha256": "9" * 64,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 409
|
||||
assert response.json()["detail"] == "Portable executor-релиз не установлен."
|
||||
assert binding.submit_count == 0
|
||||
assert queue.list_jobs() == ()
|
||||
|
||||
@@ -13,9 +13,12 @@ from k1link.observatory.portable_run_definitions import (
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||
PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA,
|
||||
PORTABLE_M49_DISPLAY_NAME,
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjectionError,
|
||||
PortableSetupProjector,
|
||||
PortableSourceCapabilityProbe,
|
||||
portable_calculation_profile_registry,
|
||||
)
|
||||
from k1link.observatory.source_admission import (
|
||||
PortableRecordedSourceCapability,
|
||||
@@ -119,6 +122,61 @@ def test_projection_uses_portable_identity_and_exact_model_presentation() -> Non
|
||||
}
|
||||
|
||||
|
||||
class _GenericProbe:
|
||||
def __init__(self, registry: PortableRunDefinitionRegistry) -> None:
|
||||
self.registry = registry
|
||||
|
||||
def probe(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> PortableRecordedSourceCapability:
|
||||
definition = self.registry.resolve(setup_id, definition_sha256)
|
||||
return _capability(
|
||||
source_session_id,
|
||||
adapter_sha256=definition.source_adapter.contract_sha256,
|
||||
)
|
||||
|
||||
|
||||
def test_generic_catalog_projects_lab_v1_and_model_free_m49_independently() -> None:
|
||||
registry = _registry()
|
||||
catalog = PortableSetupProjector(
|
||||
registry=registry,
|
||||
capability_probe=_GenericProbe(registry),
|
||||
).catalog(_source(NEW_SESSION_ID))
|
||||
|
||||
setups = {setup["setup_id"]: setup for setup in catalog["setups"]}
|
||||
assert set(setups) == {
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
"m49-tgs-portable-v2",
|
||||
}
|
||||
m49 = setups["m49-tgs-portable-v2"]
|
||||
assert m49["display_name"] == PORTABLE_M49_DISPLAY_NAME
|
||||
assert m49["run_definition"]["models"] == []
|
||||
assert m49["source_compatibility"]["outcome"] == "pass"
|
||||
assert m49["executor"]["state"] == "not-installed"
|
||||
assert m49["preflight"]["submission_allowed"] is False
|
||||
|
||||
|
||||
def test_calculation_profile_policies_cover_both_exact_portable_definitions() -> None:
|
||||
registry = _registry()
|
||||
profiles = portable_calculation_profile_registry(registry)
|
||||
|
||||
resolved = {
|
||||
definition.setup_id: profiles.resolve(definition)
|
||||
for definition in registry.definitions
|
||||
}
|
||||
assert resolved["lab-v1-eomt-ddrnet-portable-v1"].lab_id == "LAB V1"
|
||||
assert (
|
||||
resolved["lab-v1-eomt-ddrnet-portable-v1"].display_name
|
||||
== PORTABLE_LAB_V1_DISPLAY_NAME
|
||||
)
|
||||
assert resolved["m49-tgs-portable-v2"].lab_id == "LAB M4.9T5"
|
||||
assert resolved["m49-tgs-portable-v2"].display_name == PORTABLE_M49_DISPLAY_NAME
|
||||
|
||||
|
||||
def test_new_compatible_source_passes_capability_but_uninstalled_executor_blocks() -> None:
|
||||
source = _source(NEW_SESSION_ID)
|
||||
setup = _projector(lambda session_id: _capability(session_id)).project(source)
|
||||
@@ -235,10 +293,7 @@ def test_ready_executor_still_blocks_without_a_dispatch_boundary() -> None:
|
||||
assert setup["preflight"] == {
|
||||
"outcome": "blocked",
|
||||
"action": "blocked",
|
||||
"reason": (
|
||||
"Server-side проверка definition/check SHA и постановка portable "
|
||||
"LAB V1 в очередь пока недоступны."
|
||||
),
|
||||
"reason": ("Server-side проверка и постановка portable-сетапа в очередь недоступны."),
|
||||
"submission_allowed": False,
|
||||
"existing_result_ids": [],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.artifact_gateway import CentralArtifactStore
|
||||
from k1link.observatory.m49_portable_result import (
|
||||
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
validate_m49_portable_result,
|
||||
)
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
PortableObservatoryArtifactTransport,
|
||||
)
|
||||
from k1link.observatory.portable_lab_v1_executor import validate_lab_v1_result_v2
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
PortableResultContractValidatorRegistration,
|
||||
PortableResultContractValidatorRegistry,
|
||||
)
|
||||
from k1link.observatory.portable_result_publisher import (
|
||||
PortableObservatoryResultPublisher,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||
PORTABLE_LAB_V1_SETUP_ID,
|
||||
PORTABLE_M49_DISPLAY_NAME,
|
||||
PORTABLE_M49_SETUP_ID,
|
||||
)
|
||||
from k1link.observatory.portable_worker_integration import (
|
||||
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV,
|
||||
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV,
|
||||
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
|
||||
PortableWorkerIntegrationError,
|
||||
PortableWorkerStorageRoots,
|
||||
build_portable_observatory_worker_integration,
|
||||
portable_result_validator_registry,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import ObservatoryRecordedJobQueue
|
||||
from k1link.sessions import RecordedMediaInspector, SessionStore
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
|
||||
|
||||
def _definitions() -> PortableRunDefinitionRegistry:
|
||||
return PortableRunDefinitionRegistry.from_file(REGISTRY_PATH)
|
||||
|
||||
|
||||
def test_exact_validator_registry_covers_both_portable_profiles() -> None:
|
||||
definitions = _definitions()
|
||||
|
||||
validators = portable_result_validator_registry(definitions)
|
||||
|
||||
assert validators.resolve(PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256) is validate_lab_v1_result_v2
|
||||
assert validators.resolve(M49_PORTABLE_RESULT_CONTRACT_SHA256) is validate_m49_portable_result
|
||||
|
||||
|
||||
def test_validator_registry_fails_when_one_required_profile_is_absent() -> None:
|
||||
definitions = _definitions()
|
||||
only_lab_v1 = PortableRunDefinitionRegistry((definitions.definitions[0],))
|
||||
|
||||
with pytest.raises(
|
||||
PortableWorkerIntegrationError,
|
||||
match="required portable setup is unavailable",
|
||||
):
|
||||
portable_result_validator_registry(only_lab_v1)
|
||||
|
||||
|
||||
def test_server_integration_constructs_dormant_transport_and_publisher(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
definitions = _definitions()
|
||||
store = SessionStore(tmp_path / "repository")
|
||||
inspector = RecordedMediaInspector(tmp_path / "media-inspections")
|
||||
source_cas_root = tmp_path / "source-cas"
|
||||
result_staging_root = tmp_path / "result-staging"
|
||||
source_cas_root.mkdir()
|
||||
result_staging_root.mkdir()
|
||||
|
||||
integration = build_portable_observatory_worker_integration(
|
||||
queue=cast(ObservatoryRecordedJobQueue, object()),
|
||||
session_store=store,
|
||||
media_inspector=inspector,
|
||||
definitions=definitions,
|
||||
artifact_store=CentralArtifactStore(tmp_path / "artifacts", create=True),
|
||||
source_cas_root=source_cas_root,
|
||||
result_staging_root=result_staging_root,
|
||||
)
|
||||
|
||||
assert integration.supported_setup_ids == (
|
||||
PORTABLE_LAB_V1_SETUP_ID,
|
||||
PORTABLE_M49_SETUP_ID,
|
||||
)
|
||||
assert isinstance(
|
||||
integration.artifact_transport,
|
||||
PortableObservatoryArtifactTransport,
|
||||
)
|
||||
assert isinstance(
|
||||
integration.result_publisher,
|
||||
PortableObservatoryResultPublisher,
|
||||
)
|
||||
profiles = {
|
||||
policy.setup_id: policy.display_name for policy in integration.calculation_profiles.policies
|
||||
}
|
||||
assert profiles == {
|
||||
PORTABLE_LAB_V1_SETUP_ID: PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||
PORTABLE_M49_SETUP_ID: PORTABLE_M49_DISPLAY_NAME,
|
||||
}
|
||||
|
||||
|
||||
def test_server_integration_rejects_swapped_validator_identities(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
definitions = _definitions()
|
||||
swapped = PortableResultContractValidatorRegistry(
|
||||
(
|
||||
PortableResultContractValidatorRegistration(
|
||||
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
|
||||
validate_m49_portable_result,
|
||||
),
|
||||
PortableResultContractValidatorRegistration(
|
||||
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
validate_lab_v1_result_v2,
|
||||
),
|
||||
)
|
||||
)
|
||||
source_cas_root = tmp_path / "source-cas"
|
||||
result_staging_root = tmp_path / "result-staging"
|
||||
source_cas_root.mkdir()
|
||||
result_staging_root.mkdir()
|
||||
|
||||
with pytest.raises(
|
||||
PortableWorkerIntegrationError,
|
||||
match="validator registration changed identity",
|
||||
):
|
||||
build_portable_observatory_worker_integration(
|
||||
queue=cast(ObservatoryRecordedJobQueue, object()),
|
||||
session_store=SessionStore(tmp_path / "repository"),
|
||||
media_inspector=RecordedMediaInspector(tmp_path / "media-inspections"),
|
||||
definitions=definitions,
|
||||
artifact_store=CentralArtifactStore(tmp_path / "artifacts", create=True),
|
||||
validators=swapped,
|
||||
source_cas_root=source_cas_root,
|
||||
result_staging_root=result_staging_root,
|
||||
)
|
||||
|
||||
|
||||
def test_storage_roots_load_only_from_existing_disjoint_central_directories(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
boundary = tmp_path / "nodedc-mission-core"
|
||||
artifact_store = boundary / "artifact-store"
|
||||
source_cas = boundary / "observatory-worker" / "source-cas"
|
||||
result_staging = boundary / "observatory-worker" / "result-staging"
|
||||
for path in (artifact_store, source_cas, result_staging):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
roots = PortableWorkerStorageRoots.from_environment(
|
||||
artifact_store_root=artifact_store,
|
||||
environment={
|
||||
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: str(source_cas),
|
||||
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: str(result_staging),
|
||||
},
|
||||
)
|
||||
|
||||
assert roots == PortableWorkerStorageRoots(
|
||||
source_cas_root=source_cas,
|
||||
result_staging_root=result_staging,
|
||||
)
|
||||
|
||||
|
||||
def test_server_integration_has_no_data_directory_storage_fallback(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
store = SessionStore(tmp_path / "repository")
|
||||
|
||||
with pytest.raises(
|
||||
PortableWorkerIntegrationError,
|
||||
match="storage roots must be explicitly configured",
|
||||
):
|
||||
build_portable_observatory_worker_integration(
|
||||
queue=cast(ObservatoryRecordedJobQueue, object()),
|
||||
session_store=store,
|
||||
media_inspector=RecordedMediaInspector(tmp_path / "media-inspections"),
|
||||
definitions=_definitions(),
|
||||
artifact_store=CentralArtifactStore(tmp_path / "artifacts", create=True),
|
||||
)
|
||||
|
||||
assert not (store.data_dir / "observatory-worker-source-cas").exists()
|
||||
assert not (store.data_dir / "observatory-worker-result-staging").exists()
|
||||
|
||||
|
||||
def test_storage_root_loading_does_not_create_an_unavailable_mount_path(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
boundary = tmp_path / "nodedc-mission-core"
|
||||
artifact_store = boundary / "artifact-store"
|
||||
artifact_store.mkdir(parents=True)
|
||||
missing_source = boundary / "observatory-worker" / "source-cas"
|
||||
missing_result = boundary / "observatory-worker" / "result-staging"
|
||||
|
||||
with pytest.raises(
|
||||
PortableWorkerIntegrationError,
|
||||
match="portable source CAS is unavailable",
|
||||
):
|
||||
PortableWorkerStorageRoots.from_environment(
|
||||
artifact_store_root=artifact_store,
|
||||
environment={
|
||||
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: str(missing_source),
|
||||
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: str(missing_result),
|
||||
},
|
||||
)
|
||||
|
||||
assert not missing_source.exists()
|
||||
assert not missing_result.exists()
|
||||
|
||||
|
||||
def test_storage_roots_reject_escape_overlap_and_symlink(tmp_path: Path) -> None:
|
||||
boundary = tmp_path / "nodedc-mission-core"
|
||||
artifact_store = boundary / "artifact-store"
|
||||
result_staging = boundary / "observatory-worker" / "result-staging"
|
||||
outside = tmp_path / "outside"
|
||||
for path in (artifact_store, result_staging, outside):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with pytest.raises(PortableWorkerIntegrationError, match="must be inside"):
|
||||
PortableWorkerStorageRoots.from_paths(
|
||||
artifact_store_root=artifact_store,
|
||||
source_cas_root=outside,
|
||||
result_staging_root=result_staging,
|
||||
)
|
||||
|
||||
with pytest.raises(PortableWorkerIntegrationError, match="disjoint from"):
|
||||
PortableWorkerStorageRoots.from_paths(
|
||||
artifact_store_root=artifact_store,
|
||||
source_cas_root=artifact_store,
|
||||
result_staging_root=result_staging,
|
||||
)
|
||||
|
||||
source_target = boundary / "observatory-worker" / "source-target"
|
||||
source_target.mkdir(parents=True)
|
||||
source_link = boundary / "observatory-worker" / "source-link"
|
||||
source_link.symlink_to(source_target, target_is_directory=True)
|
||||
with pytest.raises(PortableWorkerIntegrationError, match="canonical directory"):
|
||||
PortableWorkerStorageRoots.from_paths(
|
||||
artifact_store_root=artifact_store,
|
||||
source_cas_root=source_link,
|
||||
result_staging_root=result_staging,
|
||||
)
|
||||
@@ -0,0 +1,574 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableExecutorAvailability,
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerAssetVerification,
|
||||
PortableWorkerExecutorAdapter,
|
||||
PortableWorkerExecutorSeal,
|
||||
PortableWorkerLocalAssetBinding,
|
||||
PortableWorkerResultDraft,
|
||||
PortableWorkerRuntimeAdmission,
|
||||
PortableWorkerRuntimePhase,
|
||||
PortableWorkerRuntimePlan,
|
||||
PortableWorkerRuntimeRegistry,
|
||||
PortableWorkerRuntimeRegistryError,
|
||||
PortableWorkerRuntimeUnavailableError,
|
||||
PortableWorkerSourceStage,
|
||||
inspect_runtime_candidate,
|
||||
)
|
||||
from k1link.observatory.worker_agent import (
|
||||
ObservatoryWorkerExecutionResult,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
SealedObservatoryRecordedJob,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFINITION_REGISTRY = (
|
||||
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
)
|
||||
RUNTIME_REGISTRY = (
|
||||
REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json"
|
||||
)
|
||||
|
||||
|
||||
def _definitions() -> PortableRunDefinitionRegistry:
|
||||
return PortableRunDefinitionRegistry.from_file(DEFINITION_REGISTRY)
|
||||
|
||||
|
||||
def _runtime() -> PortableWorkerRuntimeRegistry:
|
||||
return PortableWorkerRuntimeRegistry.from_file(
|
||||
RUNTIME_REGISTRY,
|
||||
definitions=_definitions(),
|
||||
)
|
||||
|
||||
|
||||
def _all_keys(value: object) -> set[str]:
|
||||
if isinstance(value, dict):
|
||||
return set(value) | {
|
||||
nested
|
||||
for child in value.values()
|
||||
for nested in _all_keys(child)
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return {nested for child in value for nested in _all_keys(child)}
|
||||
return set()
|
||||
|
||||
|
||||
def test_production_candidates_bind_exact_definitions_but_remain_blocked() -> None:
|
||||
registry = _runtime()
|
||||
|
||||
assert {candidate.setup_id for candidate in registry.candidates} == {
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
"m49-tgs-portable-v2",
|
||||
}
|
||||
for candidate in registry.candidates:
|
||||
assert candidate.ready is False
|
||||
assert candidate.executor is None
|
||||
assert "executor-release-unsealed" in candidate.blockers
|
||||
assert any(phase.state == "missing" for phase in candidate.phases)
|
||||
with pytest.raises(
|
||||
PortableWorkerRuntimeUnavailableError,
|
||||
match="no executor identity",
|
||||
):
|
||||
candidate.executor_identity()
|
||||
|
||||
|
||||
def test_candidate_contract_is_source_independent_and_instruction_free() -> None:
|
||||
document = json.loads(RUNTIME_REGISTRY.read_text(encoding="utf-8"))
|
||||
serialized = json.dumps(document, sort_keys=True)
|
||||
keys = _all_keys(document)
|
||||
|
||||
assert "RAVNOVES" not in serialized
|
||||
assert "source_session_id" not in serialized
|
||||
assert "filesystem" not in serialized
|
||||
assert not {"command", "commands", "argv", "env", "environment"} & keys
|
||||
assert not any("priority" in key for key in keys)
|
||||
|
||||
|
||||
def test_m49_reuses_portable_profile_generic_runner_and_travel_image() -> None:
|
||||
candidate = _runtime().resolve(
|
||||
"m49-tgs-portable-v2",
|
||||
"73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910",
|
||||
)
|
||||
bindings = {
|
||||
"m49-portable-profile": PortableWorkerLocalAssetBinding(
|
||||
asset_id="m49-portable-profile",
|
||||
file_path=REPOSITORY_ROOT / "config" / "perception" / "m49-tgs-portable-v2.json",
|
||||
),
|
||||
"m49-portable-runner-source": PortableWorkerLocalAssetBinding(
|
||||
asset_id="m49-portable-runner-source",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
/ "run_m49_tgs_portable.cpp"
|
||||
),
|
||||
),
|
||||
"m49-portable-runner-manifest": PortableWorkerLocalAssetBinding(
|
||||
asset_id="m49-portable-runner-manifest",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
/ "m49-tgs-portable-runner-source.json"
|
||||
),
|
||||
),
|
||||
"m49-portable-runner-wrapper": PortableWorkerLocalAssetBinding(
|
||||
asset_id="m49-portable-runner-wrapper",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
/ "run_m49_tgs_portable.sh"
|
||||
),
|
||||
),
|
||||
"m49-portable-smoke": PortableWorkerLocalAssetBinding(
|
||||
asset_id="m49-portable-smoke",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
/ "smoke_m49_tgs_portable.sh"
|
||||
),
|
||||
),
|
||||
"travel-tgs-image": PortableWorkerLocalAssetBinding(
|
||||
asset_id="travel-tgs-image",
|
||||
image_sha256=(
|
||||
"7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
admission = inspect_runtime_candidate(candidate, bindings)
|
||||
|
||||
assert {item.state for item in admission.assets} == {"matched"}
|
||||
assert admission.ready is False
|
||||
assert "portable-tgs-runner-unsealed" in admission.blockers
|
||||
assert "portable-camera-lidar-timeline-unimplemented" in admission.blockers
|
||||
assert "portable-result-assembler-unimplemented" in admission.blockers
|
||||
|
||||
|
||||
def test_m49_portable_runner_has_no_exact_source_or_frame_count_binding() -> None:
|
||||
source = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
/ "run_m49_tgs_portable.cpp"
|
||||
).read_text(encoding="utf-8")
|
||||
wrapper = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
/ "run_m49_tgs_portable.sh"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "RAVNOVES" not in source + wrapper
|
||||
assert "4489" not in source + wrapper
|
||||
assert "3928" not in source + wrapper
|
||||
assert "schedule.rows.size()" in source
|
||||
assert "verifySequence(sequence_dir, schedule.available_count)" in source
|
||||
assert "std::vector<float> values(point_count * 4)" in source
|
||||
assert "--network" not in wrapper
|
||||
|
||||
|
||||
def test_m49_portable_runner_source_release_is_content_addressed() -> None:
|
||||
root = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
)
|
||||
manifest = json.loads(
|
||||
(root / "m49-tgs-portable-runner-source.json").read_text(encoding="utf-8")
|
||||
)
|
||||
expected_release_sha256 = manifest.pop("source_release_sha256")
|
||||
actual_release_sha256 = hashlib.sha256(
|
||||
json.dumps(
|
||||
manifest,
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
assert actual_release_sha256 == expected_release_sha256
|
||||
assert manifest["input_contract"]["timeline_frame_count"] == "source-derived"
|
||||
assert manifest["input_contract"]["available_lidar_frame_count"] == "source-derived"
|
||||
for item in manifest["files"]:
|
||||
path = root / item["name"]
|
||||
assert path.stat().st_size == item["byte_length"]
|
||||
assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"]
|
||||
|
||||
|
||||
def test_lab_candidate_verifies_reusable_repository_assets_without_claiming_executor() -> None:
|
||||
candidate = _runtime().resolve(
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
"3692d41cec3949f348a36eb60a501fb2cd483fed1645679b0ec58061a2fc6dc2",
|
||||
)
|
||||
bindings = {
|
||||
"ddrnet-goose-image": PortableWorkerLocalAssetBinding(
|
||||
asset_id="ddrnet-goose-image",
|
||||
image_sha256=(
|
||||
"591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
|
||||
),
|
||||
),
|
||||
"ddrnet-goose-runner": PortableWorkerLocalAssetBinding(
|
||||
asset_id="ddrnet-goose-runner",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "lab_v1_vegetation_goose"
|
||||
/ "run_goose_vegetation_benchmark.py"
|
||||
),
|
||||
),
|
||||
"eomt-image": PortableWorkerLocalAssetBinding(
|
||||
asset_id="eomt-image",
|
||||
image_sha256=(
|
||||
"58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||
),
|
||||
),
|
||||
"eomt-orchestrator": PortableWorkerLocalAssetBinding(
|
||||
asset_id="eomt-orchestrator",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "Invoke-E4FullSessionSegmentation.ps1"
|
||||
),
|
||||
),
|
||||
"eomt-profile": PortableWorkerLocalAssetBinding(
|
||||
asset_id="eomt-profile",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "e3_k1_camera1_profile.json"
|
||||
),
|
||||
),
|
||||
"eomt-runner": PortableWorkerLocalAssetBinding(
|
||||
asset_id="eomt-runner",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "run_e4_full_session_segmentation.py"
|
||||
),
|
||||
),
|
||||
"vegetation-policy": PortableWorkerLocalAssetBinding(
|
||||
asset_id="vegetation-policy",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "config"
|
||||
/ "perception"
|
||||
/ "lab-v1-vegetation-mission-policy-v1.json"
|
||||
),
|
||||
),
|
||||
"vegetation-provider-map": PortableWorkerLocalAssetBinding(
|
||||
asset_id="vegetation-provider-map",
|
||||
file_path=(
|
||||
REPOSITORY_ROOT
|
||||
/ "config"
|
||||
/ "perception"
|
||||
/ "lab-v1-vegetation-provider-label-map-v1.json"
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
admission = inspect_runtime_candidate(candidate, bindings)
|
||||
states = {item.asset_id: item.state for item in admission.assets}
|
||||
|
||||
assert states["ddrnet-goose-image"] == "matched"
|
||||
assert states["ddrnet-goose-runner"] == "matched"
|
||||
assert states["eomt-image"] == "matched"
|
||||
assert states["eomt-orchestrator"] == "matched"
|
||||
assert states["eomt-profile"] == "matched"
|
||||
assert states["eomt-runner"] == "matched"
|
||||
assert states["ddrnet-portable-config"] == "missing"
|
||||
assert states["ddrnet-checkpoint"] == "missing"
|
||||
assert states["eomt-model-weights"] == "missing"
|
||||
assert admission.ready is False
|
||||
assert "ddrnet-component-port-uninstalled" in admission.blockers
|
||||
assert "worker-installation-receipt-unavailable" in admission.blockers
|
||||
|
||||
|
||||
def test_local_asset_tampering_is_reported_without_execution(tmp_path: Path) -> None:
|
||||
candidate = _runtime().resolve(
|
||||
"m49-tgs-portable-v2",
|
||||
"73611f24d70319ea1edca428726d6538a3cbad012a415cc0c1a7ecb7d9b4d910",
|
||||
)
|
||||
tampered = tmp_path / "m49-profile.json"
|
||||
tampered.write_text("{}\n", encoding="utf-8")
|
||||
|
||||
admission = inspect_runtime_candidate(
|
||||
candidate,
|
||||
{
|
||||
"m49-portable-profile": PortableWorkerLocalAssetBinding(
|
||||
asset_id="m49-portable-profile",
|
||||
file_path=tampered,
|
||||
),
|
||||
"travel-tgs-image": PortableWorkerLocalAssetBinding(
|
||||
asset_id="travel-tgs-image",
|
||||
image_sha256="7" * 64,
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
states = {item.asset_id: item.state for item in admission.assets}
|
||||
assert states["m49-portable-profile"] == "mismatched"
|
||||
assert states["travel-tgs-image"] == "mismatched"
|
||||
assert states["m49-portable-runner-source"] == "missing"
|
||||
assert states["m49-portable-runner-manifest"] == "missing"
|
||||
assert states["m49-portable-runner-wrapper"] == "missing"
|
||||
assert "asset-m49-portable-profile-mismatched" in admission.blockers
|
||||
assert "asset-travel-tgs-image-mismatched" in admission.blockers
|
||||
|
||||
|
||||
def test_runtime_registry_rejects_digest_drift_and_executable_instructions(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
document = json.loads(RUNTIME_REGISTRY.read_text(encoding="utf-8"))
|
||||
document["candidates"][0]["candidate_sha256"] = "f" * 64
|
||||
drifted = tmp_path / "drifted.json"
|
||||
drifted.write_text(json.dumps(document), encoding="utf-8")
|
||||
|
||||
with pytest.raises(PortableWorkerRuntimeRegistryError, match="digest changed"):
|
||||
PortableWorkerRuntimeRegistry.from_file(drifted, definitions=_definitions())
|
||||
|
||||
document = json.loads(RUNTIME_REGISTRY.read_text(encoding="utf-8"))
|
||||
document["candidates"][0]["command"] = "run-anything"
|
||||
unsafe = tmp_path / "unsafe.json"
|
||||
unsafe.write_text(json.dumps(document), encoding="utf-8")
|
||||
|
||||
with pytest.raises(PortableWorkerRuntimeRegistryError, match="forbids 'command'"):
|
||||
PortableWorkerRuntimeRegistry.from_file(unsafe, definitions=_definitions())
|
||||
|
||||
|
||||
def test_runtime_plan_is_identity_only_and_observation_only() -> None:
|
||||
plan = PortableWorkerRuntimePlan(
|
||||
job_id="observatory-run-" + ("a" * 32),
|
||||
adapter_id="m49-tgs-worker006-portable-v2",
|
||||
candidate_sha256="1" * 64,
|
||||
setup_id="m49-tgs-portable-v2",
|
||||
definition_sha256="2" * 64,
|
||||
source_bundle_sha256="3" * 64,
|
||||
source_capability_manifest_sha256="4" * 64,
|
||||
result_contract_sha256="5" * 64,
|
||||
phases=("source-delivery", "portable-tgs-runner"),
|
||||
).as_dict()
|
||||
|
||||
assert plan["authority"] == {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
serialized = json.dumps(plan, sort_keys=True)
|
||||
keys = _all_keys(plan)
|
||||
assert not {"command", "commands", "argv", "env", "environment", "path"} & keys
|
||||
assert not any("priority" in key for key in keys)
|
||||
assert "D:\\" not in serialized
|
||||
|
||||
|
||||
def test_ready_local_adapter_composes_only_local_ports_and_exact_job(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
definitions = _definitions()
|
||||
base_definition = definitions.resolve_setup("lab-v1-eomt-ddrnet-portable-v1")
|
||||
executor_availability = PortableExecutorAvailability(
|
||||
contour_id="worker-006",
|
||||
state="ready",
|
||||
release_id="lab-v1-portable-executor-v1",
|
||||
release_sha256="1" * 64,
|
||||
image_sha256="2" * 64,
|
||||
reason_code=None,
|
||||
reason=None,
|
||||
)
|
||||
definition_identity = base_definition.identity_document()
|
||||
definition_identity["executor"] = executor_availability.identity_document()
|
||||
definition = replace(
|
||||
base_definition,
|
||||
executor=executor_availability,
|
||||
definition_sha256=canonical_sha256(definition_identity),
|
||||
)
|
||||
|
||||
blocked = _runtime().resolve(
|
||||
"lab-v1-eomt-ddrnet-portable-v1",
|
||||
base_definition.definition_sha256,
|
||||
)
|
||||
executor_seal = PortableWorkerExecutorSeal(
|
||||
release_id="lab-v1-portable-executor-v1",
|
||||
release_sha256="1" * 64,
|
||||
image_sha256="2" * 64,
|
||||
)
|
||||
phases = tuple(
|
||||
PortableWorkerRuntimePhase(phase.phase_id, "implemented")
|
||||
for phase in blocked.phases
|
||||
)
|
||||
candidate_identity = blocked.identity_document()
|
||||
candidate_identity["definition_sha256"] = definition.definition_sha256
|
||||
candidate_identity["state"] = "ready"
|
||||
candidate_identity["executor"] = executor_seal.as_dict()
|
||||
candidate_identity["phases"] = [phase.as_dict() for phase in phases]
|
||||
candidate_identity["blockers"] = []
|
||||
candidate = replace(
|
||||
blocked,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
state="ready",
|
||||
executor=executor_seal,
|
||||
phases=phases,
|
||||
blockers=(),
|
||||
candidate_sha256=canonical_sha256(candidate_identity),
|
||||
)
|
||||
admission = PortableWorkerRuntimeAdmission(
|
||||
candidate_sha256=candidate.candidate_sha256,
|
||||
ready=True,
|
||||
blockers=(),
|
||||
assets=tuple(
|
||||
PortableWorkerAssetVerification(asset.asset_id, "matched", None)
|
||||
for asset in candidate.reusable_assets
|
||||
),
|
||||
)
|
||||
source_root = tmp_path / "source"
|
||||
result_root = tmp_path / "result"
|
||||
source_root.mkdir()
|
||||
result_root.mkdir()
|
||||
result_id = "portable-result-001"
|
||||
result_sha256 = "9" * 64
|
||||
|
||||
class SourceMaterializer:
|
||||
def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage:
|
||||
return PortableWorkerSourceStage(
|
||||
root=source_root,
|
||||
source_bundle_sha256=job.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=(
|
||||
job.source_capability_manifest_sha256
|
||||
),
|
||||
source_adapter_sha256=job.source_adapter_sha256,
|
||||
)
|
||||
|
||||
class Runner:
|
||||
def run(
|
||||
self,
|
||||
plan: PortableWorkerRuntimePlan,
|
||||
source: PortableWorkerSourceStage,
|
||||
) -> PortableWorkerResultDraft:
|
||||
assert source.root == source_root
|
||||
assert plan.definition_sha256 == definition.definition_sha256
|
||||
return PortableWorkerResultDraft(
|
||||
root=result_root,
|
||||
result_id=result_id,
|
||||
result_sha256=result_sha256,
|
||||
result_contract_sha256=candidate.result_contract_sha256,
|
||||
)
|
||||
|
||||
class Publisher:
|
||||
def publish(
|
||||
self,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
draft: PortableWorkerResultDraft,
|
||||
) -> ObservatoryWorkerExecutionResult:
|
||||
assert job.source_session_id == "20260831T120000Z_viewer_live"
|
||||
return ObservatoryWorkerExecutionResult(
|
||||
result_id=draft.result_id,
|
||||
result_sha256=draft.result_sha256,
|
||||
)
|
||||
|
||||
adapter = PortableWorkerExecutorAdapter(
|
||||
candidate=candidate,
|
||||
definition=definition,
|
||||
admission=admission,
|
||||
source_materializer=SourceMaterializer(),
|
||||
runner=Runner(),
|
||||
publisher=Publisher(),
|
||||
)
|
||||
with pytest.raises(
|
||||
PortableWorkerRuntimeUnavailableError,
|
||||
match="does not prove every candidate asset",
|
||||
):
|
||||
PortableWorkerExecutorAdapter(
|
||||
candidate=candidate,
|
||||
definition=definition,
|
||||
admission=replace(admission, assets=()),
|
||||
source_materializer=SourceMaterializer(),
|
||||
runner=Runner(),
|
||||
publisher=Publisher(),
|
||||
)
|
||||
job = SealedObservatoryRecordedJob(
|
||||
job_id="observatory-run-" + ("a" * 32),
|
||||
request_sha256="3" * 64,
|
||||
identity_sha256="4" * 64,
|
||||
submission_receipt_sha256="c" * 64,
|
||||
source_session_id="20260831T120000Z_viewer_live",
|
||||
source_catalog_sha256="5" * 64,
|
||||
source_bundle_sha256="6" * 64,
|
||||
source_capability_manifest_sha256="7" * 64,
|
||||
source_adapter_id=definition.source_adapter.adapter_id,
|
||||
source_adapter_version=definition.source_adapter.version,
|
||||
source_adapter_sha256=definition.source_adapter.contract_sha256,
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
executor_release_id=executor_seal.release_id,
|
||||
executor_identity=ObservatoryWorkerExecutorIdentity(
|
||||
release_sha256=executor_seal.release_sha256,
|
||||
image_sha256=executor_seal.image_sha256,
|
||||
model_manifest_sha256=definition.model_manifest_sha256,
|
||||
resource_profile_sha256=definition.resource_profile.profile_sha256,
|
||||
),
|
||||
model_release_ids=definition.learned_models,
|
||||
resource_profile_id=definition.resource_profile.profile_id,
|
||||
checkpoint_policy=definition.resource_profile.checkpoint_policy,
|
||||
allowed_checkpoints=definition.resource_profile.allowed_checkpoints,
|
||||
claim_generation=1,
|
||||
claim_claimed_at_utc="2026-08-31T09:00:00.000Z",
|
||||
claim_expires_at_utc="2026-08-31T09:05:00.000Z",
|
||||
claim_heartbeat_at_utc="2026-08-31T09:00:00.000Z",
|
||||
claim_renewal_count=0,
|
||||
restart_from_zero=False,
|
||||
)
|
||||
|
||||
assert adapter.execute(job) == ObservatoryWorkerExecutionResult(
|
||||
result_id=result_id,
|
||||
result_sha256=result_sha256,
|
||||
)
|
||||
|
||||
with pytest.raises(PortableWorkerRuntimeUnavailableError):
|
||||
PortableWorkerExecutorAdapter(
|
||||
candidate=blocked,
|
||||
definition=base_definition,
|
||||
admission=replace(admission, candidate_sha256=blocked.candidate_sha256),
|
||||
source_materializer=SourceMaterializer(),
|
||||
runner=Runner(),
|
||||
publisher=Publisher(),
|
||||
)
|
||||
@@ -350,6 +350,210 @@ def test_claim_is_exactly_idempotent_including_empty_result(tmp_path: Path) -> N
|
||||
)
|
||||
|
||||
|
||||
def test_claim_lease_renews_idempotently_and_requeues_expired_unstarted_job(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
clock_value = [NOW]
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=_definitions(),
|
||||
clock=lambda: clock_value[0],
|
||||
claim_lease_seconds=10,
|
||||
)
|
||||
job, _ = queue.submit(_intent(), enqueue=True)
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="lease-poll-001",
|
||||
)
|
||||
assert claim is not None
|
||||
assert claim.job.claimed_at_utc == NOW
|
||||
assert claim.job.claim_heartbeat_at_utc == NOW
|
||||
assert claim.job.claim_expires_at_utc == "2026-08-30T21:00:10.000Z"
|
||||
assert claim.job.claim_renewal_count == 0
|
||||
assert claim.job.as_dict()["claim_lease"] == {
|
||||
"claimed_at_utc": NOW,
|
||||
"expires_at_utc": "2026-08-30T21:00:10.000Z",
|
||||
"heartbeat_at_utc": NOW,
|
||||
"renewal_count": 0,
|
||||
}
|
||||
|
||||
clock_value[0] = "2026-08-30T21:00:04.000Z"
|
||||
renewed = queue.renew_claim(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
claim_generation=claim.job.claim_generation,
|
||||
heartbeat_sequence=1,
|
||||
)
|
||||
repeated = queue.renew_claim(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
claim_generation=claim.job.claim_generation,
|
||||
heartbeat_sequence=1,
|
||||
)
|
||||
assert renewed.claim_heartbeat_at_utc == clock_value[0]
|
||||
assert renewed.claim_expires_at_utc == "2026-08-30T21:00:14.000Z"
|
||||
assert renewed.claim_renewal_count == 1
|
||||
assert repeated == renewed
|
||||
with pytest.raises(ObservatoryRecordedQueueConflictError, match="contiguous"):
|
||||
queue.renew_claim(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
claim_generation=claim.job.claim_generation,
|
||||
heartbeat_sequence=3,
|
||||
)
|
||||
|
||||
clock_value[0] = "2026-08-30T21:00:14.000Z"
|
||||
recovered = queue.recover_stale_claims()
|
||||
assert [item.job_id for item in recovered] == [job.job_id]
|
||||
assert recovered[0].state == "queued"
|
||||
assert recovered[0].active_claim_token is None
|
||||
assert recovered[0].claim_expires_at_utc is None
|
||||
assert recovered[0].claim_renewal_count == 0
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||
queue.start(job.job_id, claim_token=claim.claim_token)
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||
queue.succeed(
|
||||
job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id="expired-worker-result",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
|
||||
replacement = queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="lease-poll-002",
|
||||
)
|
||||
assert replacement is not None
|
||||
assert replacement.job.job_id == job.job_id
|
||||
assert replacement.job.claim_generation == claim.job.claim_generation + 1
|
||||
assert replacement.claim_token != claim.claim_token
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="receipt"):
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="lease-poll-001",
|
||||
)
|
||||
|
||||
|
||||
def test_expired_running_claim_is_quarantined_and_stale_terminal_is_fenced(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
clock_value = [NOW]
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=_definitions(),
|
||||
clock=lambda: clock_value[0],
|
||||
claim_lease_seconds=10,
|
||||
)
|
||||
running, claim = _running_job(queue)
|
||||
|
||||
clock_value[0] = "2026-08-30T21:00:10.000Z"
|
||||
recovered = queue.recover_stale_claims()
|
||||
assert [item.job_id for item in recovered] == [running.job_id]
|
||||
quarantined = recovered[0]
|
||||
assert quarantined.state == "reconciliation-required"
|
||||
assert quarantined.terminal_code == "claim-lease-expired"
|
||||
assert quarantined.active_claim_token is None
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||
queue.succeed(
|
||||
running.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id="late-worker-result",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
assert queue.get(running.job_id).result_id is None
|
||||
assert (
|
||||
queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="blocked-after-expired-running",
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_legacy_sqlite_claim_schema_migrates_without_reusing_old_token(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job, _ = queue.submit(_intent(), enqueue=True)
|
||||
claim = queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="legacy-claim-001",
|
||||
)
|
||||
assert claim is not None
|
||||
with sqlite3.connect(queue.database_path) as connection:
|
||||
for column in (
|
||||
"claimed_at_utc",
|
||||
"claim_expires_at_utc",
|
||||
"claim_heartbeat_at_utc",
|
||||
"claim_renewal_count",
|
||||
):
|
||||
connection.execute(
|
||||
f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}"
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
migrated_queue = _queue(tmp_path)
|
||||
migrated = migrated_queue.get(job.job_id)
|
||||
assert migrated.state == "queued"
|
||||
assert migrated.claim_generation == 1
|
||||
assert migrated.active_claim_token is None
|
||||
assert migrated.claimed_at_utc is None
|
||||
assert migrated.claim_expires_at_utc is None
|
||||
assert migrated.claim_heartbeat_at_utc is None
|
||||
assert migrated.claim_renewal_count == 0
|
||||
with sqlite3.connect(migrated_queue.database_path) as connection:
|
||||
columns = {
|
||||
row[1]
|
||||
for row in connection.execute(
|
||||
"PRAGMA table_info(observatory_recorded_jobs)"
|
||||
).fetchall()
|
||||
}
|
||||
assert {
|
||||
"claimed_at_utc",
|
||||
"claim_expires_at_utc",
|
||||
"claim_heartbeat_at_utc",
|
||||
"claim_renewal_count",
|
||||
}.issubset(columns)
|
||||
replacement = migrated_queue.claim_next(
|
||||
claimant_id="recorded-worker",
|
||||
claim_request_id="legacy-claim-002",
|
||||
)
|
||||
assert replacement is not None
|
||||
assert replacement.job.claim_generation == 2
|
||||
assert replacement.claim_token != claim.claim_token
|
||||
|
||||
|
||||
def test_legacy_sqlite_running_owner_migrates_to_reconciliation(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
running, claim = _running_job(queue)
|
||||
with sqlite3.connect(queue.database_path) as connection:
|
||||
for column in (
|
||||
"claimed_at_utc",
|
||||
"claim_expires_at_utc",
|
||||
"claim_heartbeat_at_utc",
|
||||
"claim_renewal_count",
|
||||
):
|
||||
connection.execute(
|
||||
f"ALTER TABLE observatory_recorded_jobs DROP COLUMN {column}"
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
migrated_queue = _queue(tmp_path)
|
||||
migrated = migrated_queue.get(running.job_id)
|
||||
assert migrated.state == "reconciliation-required"
|
||||
assert migrated.terminal_code == "claim-lease-migration"
|
||||
assert migrated.active_claim_token is None
|
||||
with pytest.raises(ObservatoryRecordedQueueStaleClaimError, match="stale"):
|
||||
migrated_queue.succeed(
|
||||
running.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id="late-legacy-result",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
|
||||
|
||||
def test_single_worker_resource_has_only_one_recorded_owner(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
first, _ = queue.submit(_intent())
|
||||
|
||||
@@ -9,10 +9,15 @@ import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.observatory import LaboratorySetupRegistry, LaboratorySetupRegistryError
|
||||
from k1link.observatory import (
|
||||
OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||
LaboratorySetupRegistry,
|
||||
LaboratorySetupRegistryError,
|
||||
)
|
||||
from k1link.sessions import LabReplayCapability, LabSessionBinding, SessionNotFoundError
|
||||
from k1link.sessions.models import SessionSummary
|
||||
from k1link.web.observatory_api import build_observatory_router
|
||||
from k1link.web.session_api import build_session_router
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = REPOSITORY_ROOT / "config" / "observatory-laboratory-setups.json"
|
||||
@@ -163,6 +168,34 @@ def test_repository_setup_registry_keeps_real_definition_and_pre_definition_resu
|
||||
assert existing["preflight"]["submission_allowed"] is False
|
||||
|
||||
|
||||
def test_registry_attributes_only_the_exact_admitted_preserved_result() -> None:
|
||||
registry = _registry()
|
||||
projection = _rav004_projection()
|
||||
|
||||
assert registry.observatory_calculation_profile(projection) == {
|
||||
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||
"setup_id": "lab-v1-ravnoves004tree-final",
|
||||
"display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||
"origin": "existing-result",
|
||||
"definition_id": None,
|
||||
"definition_version": None,
|
||||
"definition_sha256": None,
|
||||
}
|
||||
assert registry.observatory_calculation_profile(
|
||||
replace(
|
||||
projection,
|
||||
session_id="lab-v1-vegetation-shadow-" + "0" * 64,
|
||||
)
|
||||
) is None
|
||||
assert projection.lab is not None
|
||||
assert registry.observatory_calculation_profile(
|
||||
replace(
|
||||
projection,
|
||||
lab=replace(projection.lab, provenance={}),
|
||||
)
|
||||
) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source", "reason_code"),
|
||||
[
|
||||
@@ -238,6 +271,54 @@ class _Store:
|
||||
raise SessionNotFoundError(session_id) from exc
|
||||
return SimpleNamespace(summary=summary)
|
||||
|
||||
def list_recent(
|
||||
self,
|
||||
*,
|
||||
limit: int,
|
||||
cursor: str | None,
|
||||
scope: str,
|
||||
include_capability_projections: bool = False,
|
||||
):
|
||||
del limit, cursor
|
||||
items = (
|
||||
(_rav004_projection(),)
|
||||
if scope == "laboratory" and include_capability_projections
|
||||
else ()
|
||||
)
|
||||
return SimpleNamespace(items=items, next_cursor=None)
|
||||
|
||||
|
||||
def test_session_catalog_v3_projects_the_exact_preserved_profile() -> None:
|
||||
registry = _registry()
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_session_router(
|
||||
_Store(), # type: ignore[arg-type]
|
||||
lab_calculation_profile_resolver=registry.observatory_calculation_profile,
|
||||
)
|
||||
)
|
||||
client = TestClient(app)
|
||||
|
||||
v2_lab = client.get(
|
||||
"/api/v1/observation-sessions",
|
||||
params={"scope": "laboratory", "lab_contract": "v2"},
|
||||
).json()["items"][0]["lab"]
|
||||
v3_lab = client.get(
|
||||
"/api/v1/observation-sessions",
|
||||
params={"scope": "laboratory", "lab_contract": "v3"},
|
||||
).json()["items"][0]["lab"]
|
||||
|
||||
assert "calculation_profile" not in v2_lab
|
||||
assert v3_lab["calculation_profile"] == {
|
||||
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||
"setup_id": "lab-v1-ravnoves004tree-final",
|
||||
"display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||
"origin": "existing-result",
|
||||
"definition_id": None,
|
||||
"definition_version": None,
|
||||
"definition_sha256": None,
|
||||
}
|
||||
|
||||
|
||||
def test_observatory_setup_catalog_and_preflight_are_read_only_and_fail_closed() -> None:
|
||||
app = FastAPI()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
@@ -334,6 +335,121 @@ def test_portable_source_admission_is_independent_from_session_label(
|
||||
assert capability["camera_profile"]["height"] == 600
|
||||
|
||||
|
||||
def test_admission_seals_exact_catalogued_spatial_replay_metadata(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
session_id = "20260831T000000Z_viewer_live"
|
||||
detail = _detail(session_id, "ignored-label")
|
||||
metadata_path = tmp_path / session_id / "captures/mqtt_live/mqtt.metadata.jsonl"
|
||||
metadata_path.parent.mkdir(parents=True)
|
||||
metadata_payload = b'{"offset":0,"topic":"/points"}\n'
|
||||
metadata_path.write_bytes(metadata_payload)
|
||||
detail = replace(
|
||||
detail,
|
||||
artifacts=(
|
||||
*detail.artifacts,
|
||||
SessionArtifact(
|
||||
artifact_id="raw-transport-index",
|
||||
kind="raw-transport-index",
|
||||
media_type="application/x-ndjson",
|
||||
byte_length=len(metadata_payload),
|
||||
sha256=None,
|
||||
integrity_status="verified",
|
||||
),
|
||||
),
|
||||
)
|
||||
store = _Store(tmp_path, detail)
|
||||
store.replay = replace(
|
||||
store.replay,
|
||||
artifacts=(
|
||||
*store.replay.artifacts,
|
||||
ReplayArtifact(
|
||||
artifact_id="raw-transport-index",
|
||||
path=metadata_path,
|
||||
media_type="application/x-ndjson",
|
||||
file_byte_length=len(metadata_payload),
|
||||
replay_byte_length=len(metadata_payload),
|
||||
expected_sha256=None,
|
||||
),
|
||||
),
|
||||
)
|
||||
service = RecordedK1SourceAdmissionService(
|
||||
data_dir=tmp_path,
|
||||
session_store=store, # type: ignore[arg-type]
|
||||
media_inspector=_Inspector(_manifest(session_id, tmp_path)), # type: ignore[arg-type]
|
||||
requirements=_requirements(),
|
||||
)
|
||||
|
||||
admission = service.check(session_id)
|
||||
|
||||
bundle = json.loads(admission.source_bundle)
|
||||
metadata = next(
|
||||
member
|
||||
for member in bundle["spatial_replay"]["members"]
|
||||
if member["artifact_id"] == "raw-transport-index"
|
||||
)
|
||||
assert metadata == {
|
||||
"artifact_id": "raw-transport-index",
|
||||
"media_type": "application/x-ndjson",
|
||||
"byte_length": len(metadata_payload),
|
||||
"replay_byte_length": len(metadata_payload),
|
||||
"sha256": hashlib.sha256(metadata_payload).hexdigest(),
|
||||
}
|
||||
assert "path" not in json.dumps(metadata, sort_keys=True)
|
||||
|
||||
|
||||
def test_admission_rejects_spatial_metadata_outside_session_root(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
session_id = "20260831T000000Z_viewer_live"
|
||||
detail = _detail(session_id, "ignored-label")
|
||||
(tmp_path / session_id).mkdir()
|
||||
metadata_path = tmp_path / "outside" / "mqtt.metadata.jsonl"
|
||||
metadata_path.parent.mkdir(parents=True)
|
||||
metadata_path.write_bytes(b"{}\n")
|
||||
detail = replace(
|
||||
detail,
|
||||
artifacts=(
|
||||
*detail.artifacts,
|
||||
SessionArtifact(
|
||||
artifact_id="raw-transport-index",
|
||||
kind="raw-transport-index",
|
||||
media_type="application/x-ndjson",
|
||||
byte_length=3,
|
||||
sha256=None,
|
||||
integrity_status="verified",
|
||||
),
|
||||
),
|
||||
)
|
||||
store = _Store(tmp_path, detail)
|
||||
store.replay = replace(
|
||||
store.replay,
|
||||
artifacts=(
|
||||
*store.replay.artifacts,
|
||||
ReplayArtifact(
|
||||
artifact_id="raw-transport-index",
|
||||
path=metadata_path,
|
||||
media_type="application/x-ndjson",
|
||||
file_byte_length=3,
|
||||
replay_byte_length=3,
|
||||
expected_sha256=None,
|
||||
),
|
||||
),
|
||||
)
|
||||
service = RecordedK1SourceAdmissionService(
|
||||
data_dir=tmp_path,
|
||||
session_store=store, # type: ignore[arg-type]
|
||||
media_inspector=_Inspector(_manifest(session_id, tmp_path)), # type: ignore[arg-type]
|
||||
requirements=_requirements(),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
PortableSourceAdmissionIntegrityError,
|
||||
match="escapes its admitted session root",
|
||||
):
|
||||
service.check(session_id)
|
||||
|
||||
|
||||
def test_catalog_capability_check_restores_media_without_preparing_it(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
||||
@@ -19,6 +19,7 @@ from k1link.observatory.worker_agent import (
|
||||
WORKER_006_CONTOUR_ID,
|
||||
ObservatoryWorkerAgent,
|
||||
ObservatoryWorkerAgentBusyError,
|
||||
ObservatoryWorkerCycleReport,
|
||||
ObservatoryWorkerExecutionResult,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
ObservatoryWorkerExecutorRegistration,
|
||||
@@ -105,6 +106,8 @@ class FakeTransport:
|
||||
starts: list[str] = field(default_factory=list)
|
||||
successes: list[tuple[str, str, str]] = field(default_factory=list)
|
||||
failures: list[tuple[str, str, str]] = field(default_factory=list)
|
||||
renewals: list[int] = field(default_factory=list)
|
||||
renewed: Event | None = None
|
||||
|
||||
def claim_next(
|
||||
self,
|
||||
@@ -132,6 +135,27 @@ class FakeTransport:
|
||||
self.starts.append(job_id)
|
||||
return self.queue.start(job_id, claim_token=claim_token).as_dict()
|
||||
|
||||
def renew_claim(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
claim_generation: int,
|
||||
heartbeat_sequence: int,
|
||||
) -> Mapping[str, object]:
|
||||
assert claimant_id == WORKER_006_CONTOUR_ID
|
||||
renewed = self.queue.renew_claim(
|
||||
job_id,
|
||||
claim_token=claim_token,
|
||||
claim_generation=claim_generation,
|
||||
heartbeat_sequence=heartbeat_sequence,
|
||||
).as_dict()
|
||||
self.renewals.append(heartbeat_sequence)
|
||||
if self.renewed is not None:
|
||||
self.renewed.set()
|
||||
return renewed
|
||||
|
||||
def succeed(
|
||||
self,
|
||||
*,
|
||||
@@ -228,12 +252,137 @@ def test_worker_agent_executes_one_sealed_allowlisted_job(tmp_path: Path) -> Non
|
||||
sealed_job = executor.jobs[0]
|
||||
assert sealed_job.executor_identity == _identity()
|
||||
assert sealed_job.source_bundle_sha256 == SOURCE_BUNDLE_SHA
|
||||
assert (
|
||||
sealed_job.submission_receipt_sha256
|
||||
== queue.get(job_id).submission_receipt_sha256
|
||||
)
|
||||
assert sealed_job.claim_expires_at_utc is not None
|
||||
assert sealed_job.claim_heartbeat_at_utc is not None
|
||||
assert sealed_job.claim_renewal_count == 0
|
||||
assert not hasattr(sealed_job, "command")
|
||||
assert not hasattr(sealed_job, "path")
|
||||
assert not hasattr(sealed_job, "environment")
|
||||
assert queue.get(job_id).state == "succeeded"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class BlockingExecutor:
|
||||
entered: Event
|
||||
release: Event
|
||||
|
||||
def execute(
|
||||
self,
|
||||
_job: SealedObservatoryRecordedJob,
|
||||
) -> ObservatoryWorkerExecutionResult:
|
||||
self.entered.set()
|
||||
assert self.release.wait(timeout=2)
|
||||
return ObservatoryWorkerExecutionResult(
|
||||
result_id="lab-v1-result",
|
||||
result_sha256=RESULT_SHA,
|
||||
)
|
||||
|
||||
|
||||
def _heartbeat_agent(
|
||||
transport: FakeTransport,
|
||||
executor: BlockingExecutor,
|
||||
) -> ObservatoryWorkerAgent:
|
||||
return ObservatoryWorkerAgent(
|
||||
transport=transport,
|
||||
executors=ObservatoryWorkerExecutorRegistry(
|
||||
(
|
||||
ObservatoryWorkerExecutorRegistration(
|
||||
identity=_identity(),
|
||||
adapter=executor,
|
||||
),
|
||||
)
|
||||
),
|
||||
claim_request_id_factory=lambda: "worker-006:heartbeat-cycle",
|
||||
heartbeat_interval_seconds=0.01,
|
||||
heartbeat_stop_timeout_seconds=1.0,
|
||||
)
|
||||
|
||||
|
||||
def test_worker_agent_renews_claim_through_executor_and_upload_window(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
entered = Event()
|
||||
release = Event()
|
||||
renewed = Event()
|
||||
transport = FakeTransport(queue, renewed=renewed)
|
||||
agent = _heartbeat_agent(
|
||||
transport,
|
||||
BlockingExecutor(entered=entered, release=release),
|
||||
)
|
||||
reports: list[ObservatoryWorkerCycleReport] = []
|
||||
|
||||
thread = Thread(target=lambda: reports.append(agent.run_once()))
|
||||
thread.start()
|
||||
assert entered.wait(timeout=2)
|
||||
assert renewed.wait(timeout=2)
|
||||
release.set()
|
||||
thread.join(timeout=2)
|
||||
|
||||
assert not thread.is_alive()
|
||||
assert len(reports) == 1
|
||||
assert reports[0].state == "succeeded"
|
||||
assert transport.renewals
|
||||
assert transport.renewals == list(range(1, len(transport.renewals) + 1))
|
||||
assert queue.get(job_id).state == "succeeded"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FailingRenewTransport(FakeTransport):
|
||||
renewal_attempted: Event = field(default_factory=Event)
|
||||
|
||||
def renew_claim(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
claim_generation: int,
|
||||
heartbeat_sequence: int,
|
||||
) -> Mapping[str, object]:
|
||||
assert claimant_id == WORKER_006_CONTOUR_ID
|
||||
assert job_id
|
||||
assert claim_token
|
||||
assert claim_generation == 1
|
||||
assert heartbeat_sequence == 1
|
||||
self.renewal_attempted.set()
|
||||
raise RuntimeError("synthetic heartbeat transport loss")
|
||||
|
||||
|
||||
def test_worker_agent_never_seals_result_after_heartbeat_loss(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
entered = Event()
|
||||
release = Event()
|
||||
transport = FailingRenewTransport(queue)
|
||||
agent = _heartbeat_agent(
|
||||
transport,
|
||||
BlockingExecutor(entered=entered, release=release),
|
||||
)
|
||||
reports: list[ObservatoryWorkerCycleReport] = []
|
||||
|
||||
thread = Thread(target=lambda: reports.append(agent.run_once()))
|
||||
thread.start()
|
||||
assert entered.wait(timeout=2)
|
||||
assert transport.renewal_attempted.wait(timeout=2)
|
||||
release.set()
|
||||
thread.join(timeout=2)
|
||||
|
||||
assert not thread.is_alive()
|
||||
assert len(reports) == 1
|
||||
report = reports[0]
|
||||
assert report.state == "lease-lost"
|
||||
assert report.failure_code == "claim-heartbeat-lost"
|
||||
assert transport.successes == []
|
||||
assert transport.failures == []
|
||||
assert queue.get(job_id).state == "running"
|
||||
|
||||
|
||||
def test_worker_agent_leaves_empty_queue_untouched(tmp_path: Path) -> None:
|
||||
queue = _queue(tmp_path)
|
||||
transport = FakeTransport(queue)
|
||||
@@ -299,9 +448,31 @@ def _corrupt_job_identity(payload: dict[str, object]) -> dict[str, object]:
|
||||
return changed
|
||||
|
||||
|
||||
def _remove_claim_lease(payload: dict[str, object]) -> dict[str, object]:
|
||||
changed = deepcopy(payload)
|
||||
job = changed["job"]
|
||||
assert isinstance(job, dict)
|
||||
job["claim_lease"] = None
|
||||
return changed
|
||||
|
||||
|
||||
def _corrupt_submission_receipt(payload: dict[str, object]) -> dict[str, object]:
|
||||
changed = deepcopy(payload)
|
||||
job = changed["job"]
|
||||
assert isinstance(job, dict)
|
||||
job["submission_receipt_sha256"] = "c" * 64
|
||||
return changed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutator",
|
||||
[_spoof_claimant, _inject_unknown_execution_payload, _corrupt_job_identity],
|
||||
[
|
||||
_spoof_claimant,
|
||||
_inject_unknown_execution_payload,
|
||||
_corrupt_job_identity,
|
||||
_corrupt_submission_receipt,
|
||||
_remove_claim_lease,
|
||||
],
|
||||
)
|
||||
def test_spoofed_unknown_or_corrupted_claim_is_rejected_without_execution(
|
||||
tmp_path: Path,
|
||||
@@ -370,6 +541,17 @@ class BlockingEmptyTransport:
|
||||
) -> Mapping[str, object]:
|
||||
raise AssertionError("an empty transport cannot start a job")
|
||||
|
||||
def renew_claim(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
claim_generation: int,
|
||||
heartbeat_sequence: int,
|
||||
) -> Mapping[str, object]:
|
||||
raise AssertionError("an empty transport cannot renew a job")
|
||||
|
||||
def succeed(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -2,12 +2,15 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
PortableArtifactTransportUnavailableError,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
@@ -15,6 +18,8 @@ from k1link.observatory.recorded_jobs import (
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.web.observatory_worker_api import (
|
||||
OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER,
|
||||
OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||
OBSERVATORY_WORKER_CONTOUR_HEADER,
|
||||
ObservatoryWorkerAuthentication,
|
||||
build_observatory_worker_router,
|
||||
@@ -29,6 +34,7 @@ WORKER_HEADERS = {
|
||||
}
|
||||
CLAIM_SCHEMA = "missioncore.observatory-worker-claim-request/v1"
|
||||
START_SCHEMA = "missioncore.observatory-worker-start-request/v1"
|
||||
RENEW_SCHEMA = "missioncore.observatory-worker-renew-request/v1"
|
||||
CHECKPOINT_SCHEMA = "missioncore.observatory-worker-checkpoint-request/v1"
|
||||
SUCCEED_SCHEMA = "missioncore.observatory-worker-succeed-request/v1"
|
||||
FAIL_SCHEMA = "missioncore.observatory-worker-fail-request/v1"
|
||||
@@ -74,6 +80,33 @@ def _services(tmp_path: Path) -> tuple[TestClient, ObservatoryRecordedJobQueue]:
|
||||
return TestClient(app), queue
|
||||
|
||||
|
||||
class _ArtifactTransportWithoutCompletedPackage:
|
||||
def require_completed_for_success(self, **_values: object) -> Path:
|
||||
raise PortableArtifactTransportUnavailableError("package is incomplete")
|
||||
|
||||
|
||||
def _services_with_artifact_transport(
|
||||
tmp_path: Path,
|
||||
) -> tuple[TestClient, ObservatoryRecordedJobQueue]:
|
||||
definition = _definition()
|
||||
queue = ObservatoryRecordedJobQueue(
|
||||
tmp_path,
|
||||
definitions=RecordedRunDefinitionRegistry((definition,)),
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_observatory_worker_router(
|
||||
queue,
|
||||
authentication=ObservatoryWorkerAuthentication(
|
||||
bearer_token_sha256=WORKER_TOKEN_SHA256,
|
||||
contour_id="worker-006",
|
||||
),
|
||||
artifact_transport=_ArtifactTransportWithoutCompletedPackage(), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
return TestClient(app), queue
|
||||
|
||||
|
||||
def _enqueue(
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
*,
|
||||
@@ -110,7 +143,7 @@ def _claim(
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
return response.json()
|
||||
return cast(dict[str, Any], response.json())
|
||||
|
||||
|
||||
def test_worker_authentication_requires_digest_and_configured_contour(
|
||||
@@ -280,6 +313,48 @@ def test_worker_can_start_checkpoint_and_read_job_but_stale_token_fails_closed(
|
||||
assert "active_claim_token" not in fetched.json()
|
||||
|
||||
|
||||
def test_worker_can_renew_exact_claim_lease_idempotently(tmp_path: Path) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
claim = _claim(client)
|
||||
initial_lease = claim["job"]["claim_lease"]
|
||||
assert initial_lease["renewal_count"] == 0
|
||||
request = {
|
||||
"schema_version": RENEW_SCHEMA,
|
||||
"claim_token": claim["claim_token"],
|
||||
"claim_generation": claim["job"]["claim_generation"],
|
||||
"heartbeat_sequence": 1,
|
||||
}
|
||||
|
||||
renewed = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
repeated = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew",
|
||||
headers=WORKER_HEADERS,
|
||||
json=request,
|
||||
)
|
||||
stale_generation = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew",
|
||||
headers=WORKER_HEADERS,
|
||||
json={**request, "claim_generation": request["claim_generation"] + 1},
|
||||
)
|
||||
injected = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/lease/renew",
|
||||
headers=WORKER_HEADERS,
|
||||
json={**request, "command": ["python", "untrusted.py"]},
|
||||
)
|
||||
|
||||
assert renewed.status_code == 200
|
||||
assert renewed.json()["claim_lease"]["renewal_count"] == 1
|
||||
assert repeated.status_code == 200
|
||||
assert repeated.json() == renewed.json()
|
||||
assert stale_generation.status_code == 409
|
||||
assert injected.status_code == 422
|
||||
|
||||
|
||||
def test_worker_can_publish_success_idempotently(tmp_path: Path) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
@@ -328,6 +403,59 @@ def test_worker_can_publish_success_idempotently(tmp_path: Path) -> None:
|
||||
assert conflicting_failure.status_code == 409
|
||||
|
||||
|
||||
def test_artifact_transport_blocks_success_without_completed_package(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, queue = _services_with_artifact_transport(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
claim = _claim(client)
|
||||
claim_token = claim["claim_token"]
|
||||
generation = claim["job"]["claim_generation"]
|
||||
client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/start",
|
||||
headers=WORKER_HEADERS,
|
||||
json={"schema_version": START_SCHEMA, "claim_token": claim_token},
|
||||
)
|
||||
|
||||
missing_claim_headers = client.get(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/source-materialization",
|
||||
headers=WORKER_HEADERS,
|
||||
)
|
||||
malformed_generation = client.get(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/source-materialization",
|
||||
headers={
|
||||
**WORKER_HEADERS,
|
||||
OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER: claim_token,
|
||||
OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER: "0",
|
||||
},
|
||||
)
|
||||
malformed_claim_token = client.get(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/source-materialization",
|
||||
headers={
|
||||
**WORKER_HEADERS,
|
||||
OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER: "../../not-a-claim-token",
|
||||
OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER: str(generation),
|
||||
},
|
||||
)
|
||||
blocked = client.post(
|
||||
f"/api/v1/worker/observatory/recorded-jobs/{job_id}/succeed",
|
||||
headers=WORKER_HEADERS,
|
||||
json={
|
||||
"schema_version": SUCCEED_SCHEMA,
|
||||
"claim_token": claim_token,
|
||||
"result_id": "portable-result-without-upload",
|
||||
"result_sha256": "b" * 64,
|
||||
},
|
||||
)
|
||||
|
||||
assert generation == 1
|
||||
assert missing_claim_headers.status_code == 422
|
||||
assert malformed_generation.status_code == 422
|
||||
assert malformed_claim_token.status_code == 422
|
||||
assert blocked.status_code == 409
|
||||
assert queue.get(job_id).state == "running"
|
||||
|
||||
|
||||
def test_worker_can_fail_claimed_job_idempotently(tmp_path: Path) -> None:
|
||||
client, queue = _services(tmp_path)
|
||||
job_id = _enqueue(queue)
|
||||
|
||||
@@ -2,6 +2,14 @@ from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.observatory.m49_portable_result import (
|
||||
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
validate_m49_portable_result,
|
||||
)
|
||||
from k1link.observatory.portable_lab_v1_executor import validate_lab_v1_result_v2
|
||||
from k1link.observatory.portable_worker_integration import (
|
||||
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
|
||||
)
|
||||
from k1link.web import app as app_module
|
||||
|
||||
WORKER_ROUTE_PREFIX = "/api/v1/worker/observatory"
|
||||
@@ -9,12 +17,34 @@ WORKER_ROUTE_PREFIX = "/api/v1/worker/observatory"
|
||||
|
||||
def test_worker_router_is_hard_disabled_until_lease_and_publisher_exist() -> None:
|
||||
assert app_module.OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
assert app_module.OBSERVATORY_PORTABLE_RESULT_VALIDATORS is not None
|
||||
assert (
|
||||
app_module.OBSERVATORY_PORTABLE_RESULT_VALIDATORS.resolve(
|
||||
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256
|
||||
)
|
||||
is validate_lab_v1_result_v2
|
||||
)
|
||||
assert (
|
||||
app_module.OBSERVATORY_PORTABLE_RESULT_VALIDATORS.resolve(
|
||||
M49_PORTABLE_RESULT_CONTRACT_SHA256
|
||||
)
|
||||
is validate_m49_portable_result
|
||||
)
|
||||
assert app_module.OBSERVATORY_WORKER_CLAIM_LEASE_READY is False
|
||||
assert app_module.OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY is False
|
||||
assert app_module.OBSERVATORY_WORKER_PRODUCTION_API_ENABLED is False
|
||||
assert app_module.OBSERVATORY_WORKER_DISPATCH_READY is False
|
||||
assert app_module.OBSERVATORY_WORKER_AUTHENTICATION is None
|
||||
assert app_module.OBSERVATORY_WORKER_AUTHENTICATION_ERROR is not None
|
||||
assert app_module.OBSERVATORY_WORKER_API_ERROR is not None
|
||||
assert "hard-disabled" in app_module.OBSERVATORY_WORKER_API_ERROR
|
||||
if app_module.session_artifact_gateway is None:
|
||||
assert app_module.OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None
|
||||
assert app_module.OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is not None
|
||||
assert (
|
||||
"central artifact store"
|
||||
in app_module.OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR
|
||||
)
|
||||
assert not any(
|
||||
getattr(route, "path", "").startswith(WORKER_ROUTE_PREFIX)
|
||||
for route in app_module.app.routes
|
||||
|
||||
@@ -0,0 +1,515 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA,
|
||||
PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA,
|
||||
PORTABLE_SOURCE_MATERIALIZATION_SCHEMA,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||
RESULT_DOCUMENT_ROLE,
|
||||
PortableResultArtifact,
|
||||
PortableResultPackageManifest,
|
||||
canonical_json,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import canonical_sha256
|
||||
from k1link.observatory.portable_worker_runtime import PortableWorkerResultDraft
|
||||
from k1link.observatory.worker_agent import (
|
||||
WORKER_006_CONTOUR_ID,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
SealedObservatoryRecordedJob,
|
||||
)
|
||||
from k1link.observatory.worker_http_transport import (
|
||||
ObservatoryWorkerHttpError,
|
||||
ObservatoryWorkerHttpGateway,
|
||||
)
|
||||
|
||||
JOB_ID = f"observatory-run-{'1' * 32}"
|
||||
CLAIM_TOKEN = "2" * 64
|
||||
BEARER_TOKEN = "worker-006-test-bearer-token-000001"
|
||||
NOW = "2026-08-31T11:00:00.000Z"
|
||||
|
||||
|
||||
def _job(*, bundle_sha256: str, capability_sha256: str) -> SealedObservatoryRecordedJob:
|
||||
return SealedObservatoryRecordedJob(
|
||||
job_id=JOB_ID,
|
||||
request_sha256="3" * 64,
|
||||
identity_sha256="4" * 64,
|
||||
submission_receipt_sha256="c" * 64,
|
||||
source_session_id="20260831T105500Z_viewer_live",
|
||||
source_catalog_sha256="5" * 64,
|
||||
source_bundle_sha256=bundle_sha256,
|
||||
source_capability_manifest_sha256=capability_sha256,
|
||||
source_adapter_id="sealed-session-source",
|
||||
source_adapter_version=1,
|
||||
source_adapter_sha256="6" * 64,
|
||||
setup_id="portable-lab-v1",
|
||||
definition_id="portable-lab-v1-definition",
|
||||
definition_version=1,
|
||||
definition_sha256="7" * 64,
|
||||
executor_release_id="portable-lab-v1-worker",
|
||||
executor_identity=ObservatoryWorkerExecutorIdentity(
|
||||
release_sha256="8" * 64,
|
||||
image_sha256="9" * 64,
|
||||
model_manifest_sha256="a" * 64,
|
||||
resource_profile_sha256="b" * 64,
|
||||
),
|
||||
model_release_ids=("eomt-cityscapes-large", "ddrnet-39"),
|
||||
resource_profile_id="worker006-single-gpu",
|
||||
checkpoint_policy="cooperative",
|
||||
allowed_checkpoints=("semantic-pass",),
|
||||
claim_generation=1,
|
||||
claim_claimed_at_utc=NOW,
|
||||
claim_expires_at_utc="2026-08-31T11:05:00.000Z",
|
||||
claim_heartbeat_at_utc=NOW,
|
||||
claim_renewal_count=0,
|
||||
restart_from_zero=False,
|
||||
)
|
||||
|
||||
|
||||
def _claim_response() -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"claim_token": CLAIM_TOKEN,
|
||||
"job": {"job_id": JOB_ID, "claim_generation": 1},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _cache_claim(gateway: ObservatoryWorkerHttpGateway) -> None:
|
||||
payload = gateway.claim_next(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
claim_request_id="worker-006:http-transport-test",
|
||||
)
|
||||
assert payload is not None
|
||||
|
||||
|
||||
def _source_member(
|
||||
job: SealedObservatoryRecordedJob,
|
||||
*,
|
||||
kind: str,
|
||||
payload: bytes,
|
||||
artifact_id: str | None = None,
|
||||
primary: bool = False,
|
||||
camera_epoch: int | None = None,
|
||||
camera_sequence: int | None = None,
|
||||
media_type: str,
|
||||
) -> tuple[dict[str, object], bytes]:
|
||||
sha256 = hashlib.sha256(payload).hexdigest()
|
||||
identity = {
|
||||
"job_identity_sha256": job.identity_sha256,
|
||||
"source_bundle_sha256": job.source_bundle_sha256,
|
||||
"kind": kind,
|
||||
"artifact_id": artifact_id,
|
||||
"primary": primary,
|
||||
"camera_epoch": camera_epoch,
|
||||
"camera_sequence": camera_sequence,
|
||||
"media_type": media_type,
|
||||
"byte_length": len(payload),
|
||||
"sha256": sha256,
|
||||
}
|
||||
return (
|
||||
{
|
||||
"member_id": hashlib.sha256(canonical_json(identity)).hexdigest(),
|
||||
"kind": kind,
|
||||
"media_type": media_type,
|
||||
"byte_length": len(payload),
|
||||
"sha256": sha256,
|
||||
"artifact_id": artifact_id,
|
||||
"primary": primary,
|
||||
"camera_epoch": camera_epoch,
|
||||
"camera_sequence": camera_sequence,
|
||||
},
|
||||
payload,
|
||||
)
|
||||
|
||||
|
||||
def _source_contract(
|
||||
job: SealedObservatoryRecordedJob,
|
||||
*,
|
||||
inject_path: bool = False,
|
||||
) -> tuple[dict[str, object], dict[str, bytes]]:
|
||||
rows = [
|
||||
_source_member(
|
||||
job,
|
||||
kind="source-bundle",
|
||||
payload=b"source-bundle",
|
||||
media_type="application/json",
|
||||
),
|
||||
_source_member(
|
||||
job,
|
||||
kind="source-capability",
|
||||
payload=b"source-capability",
|
||||
media_type="application/json",
|
||||
),
|
||||
_source_member(
|
||||
job,
|
||||
kind="spatial-replay",
|
||||
payload=b"sealed-raw-replay",
|
||||
artifact_id="raw-primary",
|
||||
primary=True,
|
||||
media_type="application/x-nodedc-k1mqtt",
|
||||
),
|
||||
_source_member(
|
||||
job,
|
||||
kind="spatial-replay-metadata",
|
||||
payload=b'{"offset":0,"topic":"/points"}\n',
|
||||
artifact_id="raw-transport-index",
|
||||
media_type="application/x-ndjson",
|
||||
),
|
||||
_source_member(
|
||||
job,
|
||||
kind="camera-init",
|
||||
payload=b"sealed-camera-init",
|
||||
artifact_id="recorded-camera-right",
|
||||
camera_epoch=1,
|
||||
media_type='video/mp4; codecs="avc1.641028"',
|
||||
),
|
||||
_source_member(
|
||||
job,
|
||||
kind="camera-segment",
|
||||
payload=b"sealed-camera-segment",
|
||||
artifact_id="recorded-camera-right",
|
||||
camera_epoch=1,
|
||||
camera_sequence=1,
|
||||
media_type="video/iso.segment",
|
||||
),
|
||||
]
|
||||
members = [row for row, _payload in rows]
|
||||
members.sort(key=lambda row: str(row["member_id"]))
|
||||
if inject_path:
|
||||
members[0]["path"] = "../../operator-secret"
|
||||
payloads = {str(row["member_id"]): payload for row, payload in rows}
|
||||
return (
|
||||
{
|
||||
"schema_version": PORTABLE_SOURCE_MATERIALIZATION_SCHEMA,
|
||||
"job_id": job.job_id,
|
||||
"job_identity_sha256": job.identity_sha256,
|
||||
"claim_generation": job.claim_generation,
|
||||
"source": {
|
||||
"session_id": job.source_session_id,
|
||||
"bundle_sha256": job.source_bundle_sha256,
|
||||
"capability_manifest_sha256": (
|
||||
job.source_capability_manifest_sha256
|
||||
),
|
||||
},
|
||||
"members": members,
|
||||
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
},
|
||||
payloads,
|
||||
)
|
||||
|
||||
|
||||
def test_http_gateway_materializes_only_exact_claim_bound_members(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
bundle_payload = b"source-bundle"
|
||||
capability_payload = b"source-capability"
|
||||
job = _job(
|
||||
bundle_sha256=hashlib.sha256(bundle_payload).hexdigest(),
|
||||
capability_sha256=hashlib.sha256(capability_payload).hexdigest(),
|
||||
)
|
||||
manifest, payloads = _source_contract(job)
|
||||
artifact_requests: list[httpx.Request] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/claims"):
|
||||
assert request.headers["authorization"] == f"Bearer {BEARER_TOKEN}"
|
||||
assert request.headers["x-mission-core-contour-id"] == "worker-006"
|
||||
return _claim_response()
|
||||
assert request.headers["x-mission-core-claim-token"] == CLAIM_TOKEN
|
||||
assert request.headers["x-mission-core-claim-generation"] == "1"
|
||||
artifact_requests.append(request)
|
||||
if request.url.path.endswith("/source-materialization"):
|
||||
return httpx.Response(200, json=manifest)
|
||||
member_id = request.url.path.rsplit("/", 1)[-1]
|
||||
payload = payloads[member_id]
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=payload,
|
||||
headers={
|
||||
"X-Mission-Core-Content-Sha256": hashlib.sha256(payload).hexdigest()
|
||||
},
|
||||
)
|
||||
|
||||
with ObservatoryWorkerHttpGateway(
|
||||
base_url="http://127.0.0.1:18080",
|
||||
bearer_token=BEARER_TOKEN,
|
||||
work_root=tmp_path / "worker",
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as gateway:
|
||||
_cache_claim(gateway)
|
||||
stage = gateway.materialize(job)
|
||||
|
||||
assert (stage.root / "source-bundle.json").read_bytes() == bundle_payload
|
||||
assert (stage.root / "source-capability.json").read_bytes() == capability_payload
|
||||
assert (stage.root / "mqtt.raw.k1mqtt").read_bytes() == b"sealed-raw-replay"
|
||||
assert (stage.root / "mqtt.metadata.jsonl").read_bytes() == (
|
||||
b'{"offset":0,"topic":"/points"}\n'
|
||||
)
|
||||
assert (stage.root / "camera/epoch-1/init.mp4").read_bytes() == b"sealed-camera-init"
|
||||
assert (
|
||||
stage.root / "camera/epoch-1/segments/1.m4s"
|
||||
).read_bytes() == b"sealed-camera-segment"
|
||||
persisted = json.loads(
|
||||
(stage.root / "materialization-manifest.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert persisted == manifest
|
||||
assert "path" not in json.dumps(persisted, sort_keys=True)
|
||||
assert len(artifact_requests) == 1 + len(payloads)
|
||||
|
||||
|
||||
def test_http_gateway_rejects_server_selected_source_path(tmp_path: Path) -> None:
|
||||
job = _job(
|
||||
bundle_sha256=hashlib.sha256(b"source-bundle").hexdigest(),
|
||||
capability_sha256=hashlib.sha256(b"source-capability").hexdigest(),
|
||||
)
|
||||
manifest, _payloads = _source_contract(job, inject_path=True)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/claims"):
|
||||
return _claim_response()
|
||||
return httpx.Response(200, json=manifest)
|
||||
|
||||
with ObservatoryWorkerHttpGateway(
|
||||
base_url="http://localhost:18080",
|
||||
bearer_token=BEARER_TOKEN,
|
||||
work_root=tmp_path / "worker",
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as gateway:
|
||||
_cache_claim(gateway)
|
||||
with pytest.raises(
|
||||
ObservatoryWorkerHttpError,
|
||||
match="member fields changed",
|
||||
):
|
||||
gateway.materialize(job)
|
||||
|
||||
assert not list((tmp_path / "worker").rglob("operator-secret"))
|
||||
|
||||
|
||||
def _result_package(
|
||||
tmp_path: Path,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
) -> tuple[PortableWorkerResultDraft, PortableResultPackageManifest, bytes]:
|
||||
result_id = "portable-http-result-001"
|
||||
result_payload = b'{"accepted":true}'
|
||||
artifact = PortableResultArtifact(
|
||||
role=RESULT_DOCUMENT_ROLE,
|
||||
relative_path="artifacts/result.json",
|
||||
media_type="application/json",
|
||||
byte_length=len(result_payload),
|
||||
sha256=hashlib.sha256(result_payload).hexdigest(),
|
||||
)
|
||||
created_at = "2026-08-31T11:01:00.000Z"
|
||||
job_document: dict[str, object] = {
|
||||
"job_id": job.job_id,
|
||||
"request_sha256": job.request_sha256,
|
||||
"identity_sha256": job.identity_sha256,
|
||||
"submission_receipt_sha256": job.submission_receipt_sha256,
|
||||
"claim_generation": job.claim_generation,
|
||||
}
|
||||
source_document: dict[str, object] = {}
|
||||
definition_document: dict[str, object] = {}
|
||||
result_document: dict[str, object] = {"result_id": result_id}
|
||||
identity = {
|
||||
"schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||
"created_at_utc": created_at,
|
||||
"job": job_document,
|
||||
"source": source_document,
|
||||
"run_definition": definition_document,
|
||||
"result": result_document,
|
||||
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
"artifacts": [artifact.as_dict()],
|
||||
}
|
||||
package = PortableResultPackageManifest(
|
||||
identity_sha256=canonical_sha256(identity),
|
||||
created_at_utc=created_at,
|
||||
job=job_document,
|
||||
source=source_document,
|
||||
run_definition=definition_document,
|
||||
result=result_document,
|
||||
authority=dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
artifacts=(artifact,),
|
||||
)
|
||||
root = tmp_path / "draft"
|
||||
(root / "artifacts").mkdir(parents=True)
|
||||
(root / "manifest.json").write_bytes(package.canonical_bytes)
|
||||
(root / "artifacts/result.json").write_bytes(result_payload)
|
||||
return (
|
||||
PortableWorkerResultDraft(
|
||||
root=root,
|
||||
result_id=result_id,
|
||||
result_sha256=package.manifest_sha256,
|
||||
result_contract_sha256="c" * 64,
|
||||
),
|
||||
package,
|
||||
result_payload,
|
||||
)
|
||||
|
||||
|
||||
def _result_handler(
|
||||
*,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
package: PortableResultPackageManifest,
|
||||
result_payload: bytes,
|
||||
receipt_mutator: Callable[[dict[str, object]], None] | None = None,
|
||||
) -> tuple[httpx.MockTransport, list[str]]:
|
||||
artifact = package.artifacts[0]
|
||||
member_id = hashlib.sha256(
|
||||
canonical_json(
|
||||
{
|
||||
"package_identity_sha256": package.identity_sha256,
|
||||
"artifact": artifact.as_dict(),
|
||||
}
|
||||
)
|
||||
).hexdigest()
|
||||
requests: list[str] = []
|
||||
|
||||
def plan(uploaded: bool) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA,
|
||||
"job_id": job.job_id,
|
||||
"claim_generation": job.claim_generation,
|
||||
"result_id": str(package.result["result_id"]),
|
||||
"result_sha256": package.manifest_sha256,
|
||||
"package_identity_sha256": package.identity_sha256,
|
||||
"members": [
|
||||
{
|
||||
"member_id": member_id,
|
||||
"role": artifact.role,
|
||||
"media_type": artifact.media_type,
|
||||
"byte_length": artifact.byte_length,
|
||||
"sha256": artifact.sha256,
|
||||
"uploaded": uploaded,
|
||||
}
|
||||
],
|
||||
"complete": uploaded,
|
||||
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/claims"):
|
||||
return _claim_response()
|
||||
assert request.headers["x-mission-core-claim-token"] == CLAIM_TOKEN
|
||||
assert request.headers["x-mission-core-claim-generation"] == "1"
|
||||
requests.append(request.url.path)
|
||||
if request.url.path.endswith("/manifest"):
|
||||
assert request.read() == package.canonical_bytes
|
||||
return httpx.Response(200, json=plan(False))
|
||||
if "/members/" in request.url.path:
|
||||
assert request.url.path.endswith(member_id)
|
||||
assert request.read() == result_payload
|
||||
return httpx.Response(200, json=plan(True))
|
||||
receipt_identity: dict[str, object] = {
|
||||
"schema_version": PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA,
|
||||
"job_id": job.job_id,
|
||||
"job_identity_sha256": job.identity_sha256,
|
||||
"claim_generation": job.claim_generation,
|
||||
"claim_token_sha256": hashlib.sha256(
|
||||
CLAIM_TOKEN.encode("ascii")
|
||||
).hexdigest(),
|
||||
"result_id": str(package.result["result_id"]),
|
||||
"result_sha256": package.manifest_sha256,
|
||||
"package_identity_sha256": package.identity_sha256,
|
||||
"member_count": len(package.artifacts),
|
||||
"total_bytes": sum(item.byte_length for item in package.artifacts),
|
||||
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
receipt = {
|
||||
**receipt_identity,
|
||||
"receipt_sha256": hashlib.sha256(
|
||||
canonical_json(receipt_identity)
|
||||
).hexdigest(),
|
||||
}
|
||||
if receipt_mutator is not None:
|
||||
receipt_mutator(receipt)
|
||||
return httpx.Response(200, json=receipt)
|
||||
|
||||
return httpx.MockTransport(handler), requests
|
||||
|
||||
|
||||
def test_http_gateway_uploads_atomic_package_and_verifies_receipt(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
job = _job(bundle_sha256="d" * 64, capability_sha256="e" * 64)
|
||||
draft, package, result_payload = _result_package(tmp_path, job)
|
||||
transport, requests = _result_handler(
|
||||
job=job,
|
||||
package=package,
|
||||
result_payload=result_payload,
|
||||
)
|
||||
|
||||
with ObservatoryWorkerHttpGateway(
|
||||
base_url="https://mission-core.invalid",
|
||||
bearer_token=BEARER_TOKEN,
|
||||
work_root=tmp_path / "worker",
|
||||
transport=transport,
|
||||
) as gateway:
|
||||
_cache_claim(gateway)
|
||||
result = gateway.publish(job, draft)
|
||||
|
||||
assert result.result_id == draft.result_id
|
||||
assert result.result_sha256 == draft.result_sha256
|
||||
assert any(path.endswith("/manifest") for path in requests)
|
||||
assert any("/members/" in path for path in requests)
|
||||
assert any(path.endswith("/complete") for path in requests)
|
||||
|
||||
|
||||
def test_http_gateway_rejects_changed_completion_receipt(tmp_path: Path) -> None:
|
||||
job = _job(bundle_sha256="d" * 64, capability_sha256="e" * 64)
|
||||
draft, package, result_payload = _result_package(tmp_path, job)
|
||||
|
||||
def mutate(receipt: dict[str, object]) -> None:
|
||||
receipt["job_identity_sha256"] = "f" * 64
|
||||
|
||||
transport, _requests = _result_handler(
|
||||
job=job,
|
||||
package=package,
|
||||
result_payload=result_payload,
|
||||
receipt_mutator=mutate,
|
||||
)
|
||||
with ObservatoryWorkerHttpGateway(
|
||||
base_url="http://[::1]:18080",
|
||||
bearer_token=BEARER_TOKEN,
|
||||
work_root=tmp_path / "worker",
|
||||
transport=transport,
|
||||
) as gateway:
|
||||
_cache_claim(gateway)
|
||||
with pytest.raises(
|
||||
ObservatoryWorkerHttpError,
|
||||
match="completion receipt differs",
|
||||
):
|
||||
gateway.publish(job, draft)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"http://mission-core.example",
|
||||
"http://127.0.0.1:18080/api",
|
||||
"https://user:secret@mission-core.example",
|
||||
],
|
||||
)
|
||||
def test_http_gateway_rejects_unsafe_base_urls(
|
||||
tmp_path: Path,
|
||||
base_url: str,
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="base URL|loopback"):
|
||||
ObservatoryWorkerHttpGateway(
|
||||
base_url=base_url,
|
||||
bearer_token=BEARER_TOKEN,
|
||||
work_root=tmp_path / "worker",
|
||||
transport=httpx.MockTransport(
|
||||
lambda _request: httpx.Response(500)
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,207 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
|
||||
from k1link.observatory.recorded_jobs import RecordedRunDefinition
|
||||
from k1link.observatory.worker_agent import (
|
||||
ObservatoryWorkerCycleReport,
|
||||
ObservatoryWorkerExecutionResult,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
ObservatoryWorkerExecutorRegistration,
|
||||
ObservatoryWorkerExecutorRegistry,
|
||||
SealedObservatoryRecordedJob,
|
||||
)
|
||||
from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpError
|
||||
from k1link.observatory.worker_service import (
|
||||
InstalledObservatoryWorkerService,
|
||||
ObservatoryWorkerServiceConfiguration,
|
||||
ObservatoryWorkerServiceError,
|
||||
load_observatory_worker_bearer_token,
|
||||
require_ready_executor_coverage,
|
||||
)
|
||||
|
||||
|
||||
def _definition(*, release_sha256: str = "3" * 64) -> RecordedRunDefinition:
|
||||
return RecordedRunDefinition(
|
||||
setup_id="portable-lab-v1",
|
||||
definition_id="portable-lab-v1-definition",
|
||||
definition_version=1,
|
||||
definition_sha256="1" * 64,
|
||||
source_adapter_id="sealed-session-source",
|
||||
source_adapter_version=1,
|
||||
source_adapter_sha256="2" * 64,
|
||||
executor_release_id="portable-lab-v1-worker",
|
||||
executor_release_sha256=release_sha256,
|
||||
executor_image_sha256="4" * 64,
|
||||
model_release_ids=("eomt-cityscapes-large", "ddrnet-39"),
|
||||
model_manifest_sha256="5" * 64,
|
||||
resource_profile_id="worker006-single-gpu",
|
||||
resource_profile_sha256="6" * 64,
|
||||
checkpoint_policy="cooperative",
|
||||
allowed_checkpoints=("semantic-pass",),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ReadyDefinitions:
|
||||
definitions: tuple[RecordedRunDefinition, ...]
|
||||
|
||||
def ready_recorded_definitions(self) -> tuple[RecordedRunDefinition, ...]:
|
||||
return self.definitions
|
||||
|
||||
|
||||
class _Executor:
|
||||
def execute(
|
||||
self,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
) -> ObservatoryWorkerExecutionResult:
|
||||
raise AssertionError(job)
|
||||
|
||||
|
||||
def _identity(definition: RecordedRunDefinition) -> ObservatoryWorkerExecutorIdentity:
|
||||
return ObservatoryWorkerExecutorIdentity(
|
||||
release_sha256=definition.executor_release_sha256,
|
||||
image_sha256=definition.executor_image_sha256,
|
||||
model_manifest_sha256=definition.model_manifest_sha256,
|
||||
resource_profile_sha256=definition.resource_profile_sha256,
|
||||
)
|
||||
|
||||
|
||||
def _configuration(tmp_path: Path, **overrides: object) -> ObservatoryWorkerServiceConfiguration:
|
||||
values: dict[str, object] = {
|
||||
"base_url": "http://127.0.0.1:18080",
|
||||
"bearer_token_file": tmp_path / "worker.token",
|
||||
"work_root": tmp_path / "work",
|
||||
"idle_poll_seconds": 0.05,
|
||||
"transport_backoff_seconds": 0.05,
|
||||
"max_consecutive_transport_failures": 2,
|
||||
}
|
||||
values.update(overrides)
|
||||
return ObservatoryWorkerServiceConfiguration(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_service_configuration_requires_loopback_for_plain_http(tmp_path: Path) -> None:
|
||||
with pytest.raises(ValueError, match="loopback"):
|
||||
_configuration(tmp_path, base_url="http://mission-core.internal:8000")
|
||||
|
||||
configured = ObservatoryWorkerServiceConfiguration.from_environment(
|
||||
{
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE": str(tmp_path / "worker.token"),
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT": str(tmp_path / "work"),
|
||||
}
|
||||
)
|
||||
|
||||
assert configured.base_url == "http://127.0.0.1:18080"
|
||||
|
||||
with pytest.raises(ValueError, match="failure bound"):
|
||||
_configuration(tmp_path, max_consecutive_transport_failures=2.5)
|
||||
|
||||
with pytest.raises(ValueError, match="poll interval"):
|
||||
_configuration(tmp_path, idle_poll_seconds=True)
|
||||
|
||||
|
||||
def test_worker_token_loader_requires_private_exact_ascii_file(tmp_path: Path) -> None:
|
||||
token = tmp_path / "worker.token"
|
||||
token.write_text("worker-006-test-bearer-token-000001", encoding="ascii")
|
||||
token.chmod(0o600)
|
||||
|
||||
assert load_observatory_worker_bearer_token(token) == (
|
||||
"worker-006-test-bearer-token-000001"
|
||||
)
|
||||
|
||||
token.chmod(0o644)
|
||||
with pytest.raises(ObservatoryWorkerServiceError, match="permissions"):
|
||||
load_observatory_worker_bearer_token(token)
|
||||
|
||||
link = tmp_path / "worker-link.token"
|
||||
link.symlink_to(token)
|
||||
with pytest.raises(ObservatoryWorkerServiceError, match="unavailable"):
|
||||
load_observatory_worker_bearer_token(link)
|
||||
|
||||
|
||||
def test_install_time_coverage_requires_each_ready_executor_identity() -> None:
|
||||
definition = _definition()
|
||||
definitions = cast(
|
||||
PortableRunDefinitionRegistry,
|
||||
_ReadyDefinitions((definition,)),
|
||||
)
|
||||
matching = ObservatoryWorkerExecutorRegistry(
|
||||
(ObservatoryWorkerExecutorRegistration(_identity(definition), _Executor()),)
|
||||
)
|
||||
|
||||
assert require_ready_executor_coverage(
|
||||
definitions=definitions,
|
||||
executors=matching,
|
||||
) == (_identity(definition),)
|
||||
|
||||
with pytest.raises(ObservatoryWorkerServiceError, match="exact local executor"):
|
||||
require_ready_executor_coverage(
|
||||
definitions=definitions,
|
||||
executors=ObservatoryWorkerExecutorRegistry(()),
|
||||
)
|
||||
|
||||
with pytest.raises(ObservatoryWorkerServiceError, match="no portable"):
|
||||
require_ready_executor_coverage(
|
||||
definitions=cast(
|
||||
PortableRunDefinitionRegistry,
|
||||
_ReadyDefinitions(()),
|
||||
),
|
||||
executors=matching,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeGateway:
|
||||
closed: bool = False
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeAgent:
|
||||
reports: list[ObservatoryWorkerCycleReport]
|
||||
transport_failure: bool = False
|
||||
|
||||
def run_once(self) -> ObservatoryWorkerCycleReport:
|
||||
if self.transport_failure:
|
||||
raise ObservatoryWorkerHttpError("offline")
|
||||
return self.reports.pop(0)
|
||||
|
||||
|
||||
def test_polling_service_stops_cleanly_after_an_empty_cycle(tmp_path: Path) -> None:
|
||||
stop = Event()
|
||||
gateway = _FakeGateway()
|
||||
report = ObservatoryWorkerCycleReport(
|
||||
state="empty",
|
||||
claim_request_id="worker-006:test",
|
||||
)
|
||||
service = InstalledObservatoryWorkerService(
|
||||
configuration=_configuration(tmp_path),
|
||||
gateway=cast(object, gateway), # type: ignore[arg-type]
|
||||
agent=cast(object, _FakeAgent([report])), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
service.run(stop=stop, on_cycle=lambda _: stop.set())
|
||||
|
||||
assert gateway.closed is True
|
||||
|
||||
|
||||
def test_polling_service_exits_after_bounded_transport_failures(tmp_path: Path) -> None:
|
||||
gateway = _FakeGateway()
|
||||
service = InstalledObservatoryWorkerService(
|
||||
configuration=_configuration(tmp_path),
|
||||
gateway=cast(object, gateway), # type: ignore[arg-type]
|
||||
agent=cast(object, _FakeAgent([], transport_failure=True)), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
with pytest.raises(ObservatoryWorkerServiceError, match="failure bound"):
|
||||
service.run(stop=Event())
|
||||
|
||||
assert gateway.closed is True
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import plistlib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.observatory.worker_tunnel_launchd import (
|
||||
OBSERVATORY_WORKER_TUNNEL_LABEL,
|
||||
ObservatoryWorkerTunnelPlanError,
|
||||
plan_observatory_worker_tunnel_launch_agent,
|
||||
)
|
||||
|
||||
|
||||
def _executable(path: Path) -> Path:
|
||||
path.write_text("#!/bin/sh\n", encoding="utf-8")
|
||||
path.chmod(0o700)
|
||||
return path
|
||||
|
||||
|
||||
def test_tunnel_plan_is_reverse_loopback_only_and_contains_no_credential(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
data = tmp_path / "mission-core"
|
||||
data.mkdir(mode=0o700)
|
||||
ssh = _executable(tmp_path / "ssh")
|
||||
agent = tmp_path / "worker-tunnel.plist"
|
||||
|
||||
plan = plan_observatory_worker_tunnel_launch_agent(
|
||||
data_directory=data,
|
||||
agent_path=agent,
|
||||
ssh_path=ssh,
|
||||
)
|
||||
document = plistlib.loads(plan.desired_payload)
|
||||
|
||||
assert document["Label"] == OBSERVATORY_WORKER_TUNNEL_LABEL
|
||||
assert document["ProgramArguments"][-3:] == [
|
||||
"-R",
|
||||
"127.0.0.1:18080:127.0.0.1:8000",
|
||||
"mission-gpu",
|
||||
]
|
||||
assert "127.0.0.1:18080:127.0.0.1:8000" in document["ProgramArguments"]
|
||||
assert document["KeepAlive"] is True
|
||||
assert document["RunAtLoad"] is True
|
||||
assert document["AbandonProcessGroup"] is False
|
||||
assert "token" not in plan.desired_payload.decode("utf-8").lower()
|
||||
assert plan.to_dict()["changes"]["durable_mutation_performed"] is False
|
||||
|
||||
|
||||
def test_tunnel_plan_rejects_nonprivate_data_directory(tmp_path: Path) -> None:
|
||||
data = tmp_path / "mission-core"
|
||||
data.mkdir(mode=0o755)
|
||||
data.chmod(0o755)
|
||||
|
||||
with pytest.raises(ObservatoryWorkerTunnelPlanError, match="private"):
|
||||
plan_observatory_worker_tunnel_launch_agent(
|
||||
data_directory=data,
|
||||
agent_path=tmp_path / "agent.plist",
|
||||
ssh_path=_executable(tmp_path / "ssh"),
|
||||
)
|
||||
@@ -350,7 +350,7 @@ def test_session_router_exposes_immutable_lab_provenance(tmp_path: Path) -> None
|
||||
assert_no_local_paths((item, detail), repository)
|
||||
|
||||
|
||||
def test_session_router_rolls_capability_projections_out_only_in_v2(
|
||||
def test_session_router_versions_capability_and_calculation_profile_projections(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
repository = tmp_path / "repo"
|
||||
@@ -390,7 +390,21 @@ def test_session_router_rolls_capability_projections_out_only_in_v2(
|
||||
"method": lab_method(),
|
||||
},
|
||||
)
|
||||
router = build_session_router(store)
|
||||
calculation_profile = {
|
||||
"schema_version": "missioncore.observatory-calculation-profile/v1",
|
||||
"setup_id": "lab-v1-ravnoves004tree-final",
|
||||
"display_name": "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39",
|
||||
"origin": "existing-result",
|
||||
"definition_id": None,
|
||||
"definition_version": None,
|
||||
"definition_sha256": None,
|
||||
}
|
||||
router = build_session_router(
|
||||
store,
|
||||
lab_calculation_profile_resolver=lambda summary: (
|
||||
calculation_profile if summary.session_id == canonical.session_id else None
|
||||
),
|
||||
)
|
||||
list_route = endpoint(router, "/api/v1/observation-sessions", "GET")
|
||||
|
||||
default_items = list_route(limit=20, cursor=None, scope="all")["items"]
|
||||
@@ -415,11 +429,30 @@ def test_session_router_rolls_capability_projections_out_only_in_v2(
|
||||
assert set(by_id) == {legacy.session_id, canonical.session_id}
|
||||
assert by_id[legacy.session_id]["lab"]["replay_capability"] is None
|
||||
assert by_id[canonical.session_id]["lab"]["replay_capability"] == capability.as_dict()
|
||||
assert "calculation_profile" not in by_id[legacy.session_id]["lab"]
|
||||
assert "calculation_profile" not in by_id[canonical.session_id]["lab"]
|
||||
|
||||
v3_labs = list_route(
|
||||
limit=20,
|
||||
cursor=None,
|
||||
scope="laboratory",
|
||||
lab_contract="v3",
|
||||
)["items"]
|
||||
v3_by_id = {item["id"]: item for item in v3_labs}
|
||||
assert v3_by_id[legacy.session_id]["lab"]["calculation_profile"] is None
|
||||
assert (
|
||||
v3_by_id[canonical.session_id]["lab"]["calculation_profile"]
|
||||
== calculation_profile
|
||||
)
|
||||
assert (
|
||||
v3_by_id[canonical.session_id]["lab"]["replay_capability"]
|
||||
== capability.as_dict()
|
||||
)
|
||||
|
||||
application = FastAPI()
|
||||
application.include_router(router)
|
||||
assert TestClient(application).get(
|
||||
"/api/v1/observation-sessions?lab_contract=v3"
|
||||
"/api/v1/observation-sessions?lab_contract=v4"
|
||||
).status_code == 422
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user