279 lines
10 KiB
Python
279 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from dataclasses import asdict, replace
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from k1link.compute.live_perception import LivePerceptionIngress
|
|
from k1link.perception.realtime_contract import (
|
|
REQUIRED_LAYERS,
|
|
LayerEvidence,
|
|
RealtimeBudgets,
|
|
RealtimeContractError,
|
|
ReplayMeasurements,
|
|
StreamStart,
|
|
realtime_failures,
|
|
validate_scene_layers,
|
|
)
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PROFILE = ROOT / "config/perception/k1-perception-ddrnet39-rfdetr-tgs-prototype-v1.json"
|
|
|
|
|
|
def _profile() -> dict:
|
|
return json.loads(PROFILE.read_text())
|
|
|
|
|
|
def _budgets() -> RealtimeBudgets:
|
|
return RealtimeBudgets.from_dict(_profile()["realtime_contract"]["budgets"])
|
|
|
|
|
|
def _start() -> StreamStart:
|
|
return StreamStart(
|
|
run_id="run-1",
|
|
source_id="source-a",
|
|
worker_id="worker-006",
|
|
epoch_id="epoch-1",
|
|
lease_generation=1,
|
|
profile_sha256="a" * 64,
|
|
image_sha256="b" * 64,
|
|
effective_config_sha256="c" * 64,
|
|
calibration_sha256="d" * 64,
|
|
clock_domain_id="source-monotonic",
|
|
input_mode="recorded-source-paced",
|
|
)
|
|
|
|
|
|
def _measurements() -> ReplayMeasurements:
|
|
return ReplayMeasurements(
|
|
source_frames=100,
|
|
declared_source_gap_frames=0,
|
|
terminal_frames=100,
|
|
fresh_complete_scenes=100,
|
|
capacity_drops=0,
|
|
expired_frames=0,
|
|
failed_frames=0,
|
|
unexpected_unavailable_layers=0,
|
|
full_source_preloaded=False,
|
|
results_before_end_of_source=True,
|
|
replay_speed=1.0,
|
|
maximum_release_lag_ms=10.0,
|
|
output_age_p95_ms=80.0,
|
|
output_age_p99_ms=100.0,
|
|
maximum_layer_age_ms=110.0,
|
|
first_result_ms=500.0,
|
|
warmup_seconds=20.0,
|
|
stop_seconds=1.0,
|
|
peak_inflight_bytes=2_000_000,
|
|
peak_pending_camera_frames=2,
|
|
peak_vram_mib=12000,
|
|
peak_rss_mib=4000,
|
|
clock_uncertainty_ms=1.0,
|
|
backlog_growth_ms=0.0,
|
|
computed_layers=REQUIRED_LAYERS,
|
|
)
|
|
|
|
|
|
def _layers() -> tuple[LayerEvidence, ...]:
|
|
return tuple(
|
|
LayerEvidence(name, "epoch-1", 0, 0, "current", "f" * 64) for name in REQUIRED_LAYERS
|
|
)
|
|
|
|
|
|
def test_handshake_needs_no_source_size_duration_or_end() -> None:
|
|
value = _start()
|
|
assert StreamStart.from_dict(value.to_dict()) == value
|
|
assert len(json.dumps(value.to_dict()).encode()) < _budgets().maximum_start_metadata_bytes
|
|
assert replace(value, input_mode="live").channels == value.channels
|
|
assert replace(value, worker_id="worker-007").worker_id == "worker-007"
|
|
|
|
|
|
def test_existing_raw_first_ingress_delivers_without_session_end() -> None:
|
|
ingress = LivePerceptionIngress()
|
|
try:
|
|
ingress.begin_session("synthetic-source")
|
|
ingress.open_consumer("synthetic-worker")
|
|
assert ingress.take_next("synthetic-worker", timeout=0).modality == "control"
|
|
assert ingress.publish(
|
|
modality="camera-frame",
|
|
source_id="camera",
|
|
source_sequence=0,
|
|
captured_at_epoch_ns=1,
|
|
received_monotonic_ns=1,
|
|
payload=b"synthetic-frame",
|
|
)
|
|
event = ingress.take_next("synthetic-worker", timeout=0)
|
|
assert event is not None and event.payload == b"synthetic-frame"
|
|
assert ingress.snapshot()["active"] is True
|
|
finally:
|
|
ingress.close()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"field",
|
|
[
|
|
"total_bytes",
|
|
"frame_count",
|
|
"source_duration_ns",
|
|
"source_bundle_sha256",
|
|
"source_archive",
|
|
"source_members",
|
|
"wait_for_eof",
|
|
"command",
|
|
],
|
|
)
|
|
def test_full_recording_and_execution_fields_are_not_start_requirements(field: str) -> None:
|
|
value = _start().to_dict()
|
|
value[field] = "not-an-input-to-the-stream-handshake"
|
|
with pytest.raises(RealtimeContractError, match="fields"):
|
|
StreamStart.from_dict(value)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change",
|
|
[
|
|
{"lease_generation": 0},
|
|
{"lease_generation": True},
|
|
{"epoch_id": ""},
|
|
{"image_sha256": "latest"},
|
|
{"input_mode": "offline-batch"},
|
|
{"channels": ("camera",)},
|
|
{"calibration_sha256": None},
|
|
],
|
|
)
|
|
def test_invalid_handshakes_fail_closed(change: dict) -> None:
|
|
with pytest.raises(RealtimeContractError):
|
|
replace(_start(), **change)
|
|
|
|
|
|
def test_scene_reports_all_layers_before_any_end_marker_exists() -> None:
|
|
validate_scene_layers(_layers(), epoch_id="epoch-1", source_time_ns=0, maximum_layer_age_ms=250)
|
|
|
|
|
|
@pytest.mark.parametrize("layers", [_layers()[:-1], _layers()[:-1] + (_layers()[0],)])
|
|
def test_missing_or_duplicate_layer_is_not_a_complete_profile(layers: tuple) -> None:
|
|
with pytest.raises(RealtimeContractError, match="exactly once"):
|
|
validate_scene_layers(
|
|
layers, epoch_id="epoch-1", source_time_ns=0, maximum_layer_age_ms=250
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change,reason",
|
|
[
|
|
({"epoch_id": "epoch-old"}, "another stream epoch"),
|
|
({"source_time_ns": 2_000_000_000}, "future"),
|
|
({"source_time_ns": 0, "state": "held"}, "expired"),
|
|
],
|
|
)
|
|
def test_cross_epoch_future_and_expired_evidence_rejected(change: dict, reason: str) -> None:
|
|
layers = tuple(replace(x, source_time_ns=1_000_000_000) for x in _layers())
|
|
layers = (replace(layers[0], **change),) + layers[1:]
|
|
with pytest.raises(RealtimeContractError, match=reason):
|
|
validate_scene_layers(
|
|
layers, epoch_id="epoch-1", source_time_ns=1_000_000_000, maximum_layer_age_ms=250
|
|
)
|
|
|
|
|
|
def test_explicit_missing_lidar_does_not_invent_geometry_or_distance() -> None:
|
|
missing = LayerEvidence("geometry", "epoch-1", None, None, "unavailable", None)
|
|
layers = tuple(missing if x.layer == "geometry" else x for x in _layers())
|
|
validate_scene_layers(layers, epoch_id="epoch-1", source_time_ns=0, maximum_layer_age_ms=250)
|
|
with pytest.raises(RealtimeContractError, match="cannot claim"):
|
|
replace(missing, payload_sha256="e" * 64)
|
|
|
|
|
|
def test_synthetic_receipt_passes_only_whole_path_limits() -> None:
|
|
assert realtime_failures(_measurements(), _budgets()) == ()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change,reason",
|
|
[
|
|
({"full_source_preloaded": True}, "full-source-preload"),
|
|
({"results_before_end_of_source": False}, "no-incremental-result"),
|
|
({"replay_speed": 0.5}, "source-clock-not-1x"),
|
|
({"terminal_frames": 101}, "incomplete-input-accounting"),
|
|
({"fresh_complete_scenes": 50}, "incomplete-fresh-scenes"),
|
|
({"capacity_drops": 1}, "capacity-drops"),
|
|
({"expired_frames": 1}, "expired-frames"),
|
|
({"failed_frames": 1}, "failed-frames"),
|
|
({"unexpected_unavailable_layers": 1}, "required-layer-unavailable"),
|
|
({"computed_layers": ("segmentation",)}, "full-profile-not-computed"),
|
|
({"peak_pending_camera_frames": 3}, "peak_pending_camera_frames"),
|
|
({"peak_inflight_bytes": 20_000_000}, "peak_inflight_bytes"),
|
|
({"clock_uncertainty_ms": 6}, "clock_uncertainty_ms"),
|
|
({"backlog_growth_ms": 30}, "backlog_growth_ms"),
|
|
({"output_age_p99_ms": 130}, "output_age_p99_ms"),
|
|
({"maximum_layer_age_ms": 300}, "maximum_layer_age_ms"),
|
|
({"warmup_seconds": 121}, "warmup_seconds"),
|
|
({"stop_seconds": 6}, "stop_seconds"),
|
|
],
|
|
)
|
|
def test_fast_model_or_complete_archive_does_not_pass_realtime(change: dict, reason: str) -> None:
|
|
assert reason in realtime_failures(replace(_measurements(), **change), _budgets())
|
|
|
|
|
|
def test_source_gaps_stay_visible_and_are_not_model_capacity_drops() -> None:
|
|
value = replace(_measurements(), declared_source_gap_frames=10, fresh_complete_scenes=90)
|
|
assert realtime_failures(value, _budgets()) == ()
|
|
assert "incomplete-fresh-scenes" in realtime_failures(
|
|
replace(value, fresh_complete_scenes=100), _budgets()
|
|
)
|
|
assert "no-evaluable-scene" in realtime_failures(
|
|
replace(value, declared_source_gap_frames=100, fresh_complete_scenes=0), _budgets()
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change",
|
|
[
|
|
{"output_age_p99_ms": float("nan")},
|
|
{"clock_uncertainty_ms": float("inf")},
|
|
{"peak_vram_mib": -1},
|
|
{"source_frames": True},
|
|
{"full_source_preloaded": 0},
|
|
],
|
|
)
|
|
def test_non_finite_or_ambiguous_measurements_rejected(change: dict) -> None:
|
|
with pytest.raises(RealtimeContractError):
|
|
replace(_measurements(), **change)
|
|
|
|
|
|
def test_budget_schema_rejects_silent_fields_and_inconsistent_limits() -> None:
|
|
with pytest.raises(RealtimeContractError):
|
|
RealtimeBudgets.from_dict({**asdict(_budgets()), "allow_full_download": True})
|
|
with pytest.raises(RealtimeContractError):
|
|
replace(_budgets(), maximum_chunk_bytes=100_000_000)
|
|
|
|
|
|
def test_profile_pins_reused_assets_without_installing_a_fake_ready_executor() -> None:
|
|
profile = _profile()
|
|
assert profile["packaging"]["image_sha256"] is None
|
|
assert set(profile["components"]) == set(REQUIRED_LAYERS)
|
|
assert profile["stream_policy"]["inference_stride"] == 1
|
|
assert profile["input_contract"]["full_source_transfer_required"] is False
|
|
assert profile["authority"]["actuation_allowed"] is False
|
|
assert profile["components"]["geometry"]["generic_static_class"] == "static.unknown"
|
|
assert (
|
|
profile["components"]["policy"]["fine_road_sidewalk_bikeway_distinction_required"] is False
|
|
)
|
|
for ref in profile["reference_files"]:
|
|
assert hashlib.sha256((ROOT / ref["path"]).read_bytes()).hexdigest() == ref["sha256"]
|
|
registry = (ROOT / "config/observatory-portable-run-definitions.json").read_text()
|
|
assert profile["profile_id"] not in registry
|
|
|
|
|
|
def test_experimental_overload_does_not_become_realtime_qualification() -> None:
|
|
policy = _profile()["experiment_policy"]
|
|
assert policy["bounded_lab_overload_allowed"] is True
|
|
assert policy["retain_profiles_that_fail_current_hardware_realtime"] is True
|
|
assert policy["realtime_failure_blocks_experimental_packaging"] is False
|
|
assert policy["overload_is_realtime_pass"] is False
|
|
measurements = replace(_measurements(), output_age_p95_ms=150.0, output_age_p99_ms=200.0)
|
|
failures = realtime_failures(measurements, _budgets())
|
|
assert "output_age_p95_ms" in failures and "output_age_p99_ms" in failures
|