feat(perception): gate streaming lifecycle on controller readiness
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,380 @@
|
||||
"""Small synthetic control-plane checks; no models, GPU, network or host setters."""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.perception.graph_contracts import GraphState
|
||||
from k1link.perception.realtime_contract import RealtimeContractError, StreamStart
|
||||
from k1link.perception.streaming_lifecycle import StreamingLifecycle
|
||||
from k1link.perception.streaming_queue import StreamMailbox
|
||||
from k1link.perception.worker_lease import WorkerLeaseError
|
||||
from k1link.perception.worker_operating_envelope import WorkerOperatingEnvelope, WorkerSnapshot
|
||||
from k1link.perception.worker_readiness import WorkerReadinessError, WorkerReadinessMonitor
|
||||
|
||||
|
||||
def binding(generation=1):
|
||||
return StreamStart(
|
||||
f"run-{generation}",
|
||||
"fixture",
|
||||
"worker-006",
|
||||
f"epoch-{generation}",
|
||||
generation,
|
||||
"a" * 64,
|
||||
"b" * 64,
|
||||
"c" * 64,
|
||||
"d" * 64,
|
||||
"source-clock",
|
||||
"recorded-source-paced",
|
||||
)
|
||||
|
||||
|
||||
def observation(now=1_000_000_000, **changes):
|
||||
return replace(
|
||||
WorkerSnapshot(
|
||||
"worker-006",
|
||||
"worker-clock",
|
||||
now,
|
||||
"RTX 4090",
|
||||
"610.47",
|
||||
"b" * 64,
|
||||
"c" * 64,
|
||||
8000,
|
||||
8192,
|
||||
2610,
|
||||
10251,
|
||||
"run-1",
|
||||
1,
|
||||
(),
|
||||
True,
|
||||
),
|
||||
**changes,
|
||||
)
|
||||
|
||||
|
||||
def monitor(*, initial=None, mode="strict-envelope", start=None):
|
||||
return WorkerReadinessMonitor(
|
||||
start or binding(),
|
||||
WorkerOperatingEnvelope("fixture/v1", "RTX 4090", "610.47", 8000, 8192, 2610, 10251),
|
||||
initial or observation(),
|
||||
mode=mode,
|
||||
clock_domain_id="worker-clock",
|
||||
now_monotonic_ns=1_000_000_000,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def controlled(tmp_path):
|
||||
runs = []
|
||||
|
||||
def create(**options):
|
||||
clock = [1_000_000_000]
|
||||
readiness = monitor(**options)
|
||||
run = StreamingLifecycle(
|
||||
binding(),
|
||||
tmp_path / "worker",
|
||||
StreamMailbox(),
|
||||
threading.Event(),
|
||||
readiness=readiness,
|
||||
clock_ns=lambda: clock[0],
|
||||
)
|
||||
runs.append(run)
|
||||
return run, clock
|
||||
|
||||
yield create
|
||||
for run in runs:
|
||||
assert run.close()
|
||||
|
||||
|
||||
def test_admission_requires_fresh_exclusive_inventory_before_any_lease_or_spawn(tmp_path):
|
||||
with pytest.raises(WorkerReadinessError, match="competing-gpu"):
|
||||
StreamingLifecycle(
|
||||
binding(),
|
||||
tmp_path / "worker",
|
||||
StreamMailbox(),
|
||||
threading.Event(),
|
||||
readiness=monitor(initial=observation(competing_gpu_clients=("other-model",))),
|
||||
)
|
||||
assert not (tmp_path / "worker").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("warmup", [False, None])
|
||||
def test_warmup_may_start_without_ready_but_cannot_admit_frames(controlled, warmup):
|
||||
run, _ = controlled(initial=observation(warmup_complete=warmup, sm_clock_mhz=210))
|
||||
assert run.state == GraphState.STARTING
|
||||
with pytest.raises(WorkerLeaseError, match="accepting"):
|
||||
run.admit(run.start, {"sequence": 0, "payload_bytes": 1})
|
||||
with pytest.raises(WorkerReadinessError, match="warmup-not-complete"):
|
||||
run.ready()
|
||||
assert run.reason == "worker-not-ready"
|
||||
|
||||
|
||||
def test_warmup_transition_then_frames_and_results_use_same_monitor(controlled):
|
||||
run, clock = controlled(initial=observation(warmup_complete=False, sm_clock_mhz=210))
|
||||
clock[0] += 100_000_000
|
||||
run.observe_worker(run.start, observation(clock[0]))
|
||||
run.ready()
|
||||
with run.work(run.start, "gpu"):
|
||||
pass
|
||||
run.validate_result_binding(run.start.to_dict())
|
||||
audit = run.snapshot()["worker_readiness"]
|
||||
assert audit["observations"] == 2
|
||||
assert audit["warmup_readiness_checked"]
|
||||
assert audit["post_warmup_envelope_violations"] == []
|
||||
assert audit["realtime_qualified"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["strict-envelope", "labelled-experiment"])
|
||||
@pytest.mark.parametrize(
|
||||
"change,reason",
|
||||
[
|
||||
({"worker_id": "worker-007"}, "worker_id-mismatch"),
|
||||
({"image_sha256": None}, "image_sha256-mismatch"),
|
||||
({"effective_config_sha256": "f" * 64}, "effective_config_sha256-mismatch"),
|
||||
({"gpu_owner_run_id": "other"}, "exclusive-worker-lease"),
|
||||
({"lease_generation": 2}, "exclusive-worker-lease"),
|
||||
({"competing_gpu_clients": ("other-model",)}, "competing-gpu"),
|
||||
({"competing_gpu_clients": None}, "competing-gpu"),
|
||||
({"warmup_complete": None}, "warmup-not-complete"),
|
||||
],
|
||||
)
|
||||
def test_hard_failures_stop_even_labelled_experiments(controlled, mode, change, reason):
|
||||
run, clock = controlled(mode=mode)
|
||||
run.ready()
|
||||
clock[0] += 100_000_000
|
||||
with pytest.raises(WorkerReadinessError, match=reason):
|
||||
run.observe_worker(run.start, observation(clock[0], **change))
|
||||
assert run.stop_event.is_set() and run.state == GraphState.STOPPING
|
||||
assert run.reason == "worker-not-ready" and not run.lease.released
|
||||
with pytest.raises(WorkerLeaseError):
|
||||
run.validate_result_binding(run.start.to_dict())
|
||||
|
||||
|
||||
def test_performance_failure_is_allowed_only_in_explicit_experiment_and_is_latched(controlled):
|
||||
run, clock = controlled(mode="labelled-experiment")
|
||||
run.ready()
|
||||
clock[0] += 100_000_000
|
||||
run.observe_worker(run.start, observation(clock[0], memory_clock_mhz=405))
|
||||
run.renew(run.start)
|
||||
assert not run.stop_event.is_set()
|
||||
clock[0] += 100_000_000
|
||||
run.observe_worker(run.start, observation(clock[0]))
|
||||
audit = run.snapshot()["worker_readiness"]
|
||||
assert audit["current_failures"] == []
|
||||
assert audit["post_warmup_envelope_violations"] == [
|
||||
"memory_clock_mhz-below-envelope-or-unknown"
|
||||
]
|
||||
assert audit["realtime_qualified"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("initial", [True, False])
|
||||
def test_strict_envelope_rejects_low_clocks_at_ready_or_during_run(controlled, initial):
|
||||
run, clock = controlled(initial=observation(memory_clock_mhz=405) if initial else None)
|
||||
if initial:
|
||||
with pytest.raises(WorkerReadinessError, match="memory_clock_mhz-below"):
|
||||
run.ready()
|
||||
else:
|
||||
run.ready()
|
||||
clock[0] += 100_000_000
|
||||
with pytest.raises(WorkerReadinessError, match="memory_clock_mhz-below"):
|
||||
run.observe_worker(run.start, observation(clock[0], memory_clock_mhz=405))
|
||||
assert run.reason == "worker-not-ready"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["renew", "admit", "result", "refresh"])
|
||||
def test_heartbeats_cannot_extend_inventory_or_late_refresh_revive_it(controlled, operation):
|
||||
run, clock = controlled(mode="labelled-experiment")
|
||||
run.ready()
|
||||
clock[0] += 900_000_000
|
||||
run.renew(run.start) # Lease now lasts beyond the inventory deadline.
|
||||
clock[0] += 100_000_001
|
||||
with pytest.raises(WorkerReadinessError, match="worker-snapshot-expired"):
|
||||
if operation == "renew":
|
||||
run.renew(run.start)
|
||||
elif operation == "admit":
|
||||
run.admit(run.start, {"sequence": 0, "payload_bytes": 1})
|
||||
elif operation == "result":
|
||||
run.validate_result_binding(run.start.to_dict())
|
||||
else:
|
||||
run.observe_worker(run.start, observation(clock[0]))
|
||||
assert run.reason == "worker-not-ready"
|
||||
assert run.lease.renewals == 1
|
||||
|
||||
|
||||
def test_fresh_inventory_does_not_renew_lease(controlled):
|
||||
run, clock = controlled()
|
||||
run.ready()
|
||||
for _ in range(2):
|
||||
clock[0] += 900_000_000
|
||||
run.observe_worker(run.start, observation(clock[0]))
|
||||
assert run.lease.renewals == 0
|
||||
clock[0] += 200_000_000
|
||||
with pytest.raises(WorkerLeaseError):
|
||||
run.observe_worker(run.start, observation(clock[0]))
|
||||
assert run.reason == "lease-lost"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value", [("epoch_id", "old"), ("lease_generation", 2), ("source_id", "old")]
|
||||
)
|
||||
def test_old_controller_update_cannot_change_or_stop_current_owner(controlled, field, value):
|
||||
run, clock = controlled()
|
||||
run.ready()
|
||||
before = run.snapshot()["worker_readiness"]
|
||||
clock[0] += 100_000_000
|
||||
with pytest.raises(WorkerLeaseError, match="mismatch"):
|
||||
run.observe_worker(
|
||||
replace(run.start, **{field: value}), observation(clock[0], competing_gpu_clients=None)
|
||||
)
|
||||
assert run.snapshot()["worker_readiness"] == before
|
||||
assert run.state == GraphState.RUNNING and not run.stop_event.is_set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"observed_monotonic_ns": 1_000_000_000},
|
||||
{"observed_monotonic_ns": 1_200_000_000},
|
||||
{"clock_domain_id": "mac-clock"},
|
||||
],
|
||||
)
|
||||
def test_bad_telemetry_order_and_clock_fail_closed(controlled, change):
|
||||
run, clock = controlled()
|
||||
run.ready()
|
||||
clock[0] += 100_000_000
|
||||
with pytest.raises(WorkerReadinessError):
|
||||
run.observe_worker(run.start, replace(observation(clock[0]), **change))
|
||||
assert run.reason == "worker-not-ready"
|
||||
|
||||
|
||||
def test_idle_watchdog_stops_owned_child_on_inventory_expiry_without_another_frame(controlled):
|
||||
run, clock = controlled()
|
||||
child = run.spawn(
|
||||
lambda: subprocess.Popen(
|
||||
[sys.executable, "-c", "import time; print('ready', flush=True); time.sleep(10)"],
|
||||
stdout=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
)
|
||||
)
|
||||
try:
|
||||
assert child.stdout.readline() == b"ready\n"
|
||||
run.ready()
|
||||
clock[0] += 1_000_000_001 # Lease still live; no frame or heartbeat to trigger a check.
|
||||
assert run.stop_event.wait(1)
|
||||
child.wait(timeout=2)
|
||||
assert run.reason == "worker-not-ready" and not run.lease.released
|
||||
finally:
|
||||
assert run.close()
|
||||
child.stdout.close()
|
||||
|
||||
|
||||
def test_loss_during_compute_rejects_return_and_keeps_lease_until_buffers_released(controlled):
|
||||
run, clock = controlled()
|
||||
run.ready()
|
||||
packet = {"sequence": 0, "payload_bytes": 10}
|
||||
run.admit(run.start, packet)
|
||||
assert run.mailbox.take() is packet
|
||||
try:
|
||||
with pytest.raises(WorkerLeaseError), run.work(run.start, "gpu"):
|
||||
clock[0] += 1_000_000_001
|
||||
with pytest.raises(WorkerLeaseError):
|
||||
run.check_current(run.start)
|
||||
assert run.reason == "worker-not-ready"
|
||||
assert not run.close() # Callback and input still owned.
|
||||
finally:
|
||||
run.mailbox.release(packet)
|
||||
assert run.close()
|
||||
assert run.state == GraphState.FAILED
|
||||
|
||||
|
||||
def test_observation_loss_during_compute_blocks_return_and_preserves_owned_input(controlled):
|
||||
run, clock = controlled()
|
||||
run.ready()
|
||||
with pytest.raises(WorkerLeaseError), run.work(run.start, "cpu"):
|
||||
clock[0] += 100_000_000
|
||||
with pytest.raises(WorkerReadinessError):
|
||||
run.observe_worker(run.start, observation(clock[0], competing_gpu_clients=None))
|
||||
assert not run.close()
|
||||
assert run.close()
|
||||
|
||||
|
||||
def test_unmonitored_legacy_runtime_cannot_report_monitored_readiness(tmp_path):
|
||||
run = StreamingLifecycle(binding(), tmp_path, StreamMailbox(), threading.Event())
|
||||
try:
|
||||
assert run.snapshot()["worker_readiness"] == {"enabled": False, "realtime_qualified": False}
|
||||
with pytest.raises(WorkerReadinessError, match="not configured"):
|
||||
run.observe_worker(run.start, observation())
|
||||
finally:
|
||||
assert run.close()
|
||||
|
||||
|
||||
def test_invalid_mode_and_binding_rejected_without_mutating_monitor():
|
||||
with pytest.raises(RealtimeContractError):
|
||||
monitor(mode="automatic-pass")
|
||||
value = monitor()
|
||||
before = value.snapshot()
|
||||
with pytest.raises(WorkerReadinessError, match="mismatch"):
|
||||
value.check(binding(2), now_monotonic_ns=1_000_000_000, require_warmup=True)
|
||||
assert value.snapshot() == before
|
||||
|
||||
|
||||
def test_failed_monitor_is_terminal_even_when_newer_healthy_snapshot_arrives():
|
||||
value = monitor()
|
||||
with pytest.raises(WorkerReadinessError):
|
||||
value.check(binding(), now_monotonic_ns=2_000_000_001, require_warmup=True)
|
||||
with pytest.raises(WorkerReadinessError, match="expired"):
|
||||
value.observe(
|
||||
binding(),
|
||||
observation(2_000_000_002),
|
||||
now_monotonic_ns=2_000_000_002,
|
||||
require_warmup=True,
|
||||
)
|
||||
assert value.snapshot()["observations"] == 1
|
||||
|
||||
|
||||
def test_only_verified_cleanup_allows_next_generation_with_new_monitor(controlled, tmp_path):
|
||||
run, clock = controlled()
|
||||
run.ready()
|
||||
packet = {"sequence": 0, "payload_bytes": 10}
|
||||
run.admit(run.start, packet)
|
||||
assert run.mailbox.take() is packet
|
||||
clock[0] += 100_000_000
|
||||
with pytest.raises(WorkerReadinessError):
|
||||
run.observe_worker(run.start, observation(clock[0], competing_gpu_clients=None))
|
||||
next_start = binding(2)
|
||||
next_monitor = monitor(
|
||||
start=next_start,
|
||||
initial=observation(gpu_owner_run_id=next_start.run_id, lease_generation=2),
|
||||
)
|
||||
try:
|
||||
assert not run.close()
|
||||
with pytest.raises(WorkerLeaseError, match="already owned"):
|
||||
StreamingLifecycle(
|
||||
next_start,
|
||||
tmp_path / "worker",
|
||||
StreamMailbox(),
|
||||
threading.Event(),
|
||||
readiness=next_monitor,
|
||||
clock_ns=lambda: clock[0],
|
||||
)
|
||||
finally:
|
||||
run.mailbox.release(packet)
|
||||
assert run.close()
|
||||
next_run = StreamingLifecycle(
|
||||
next_start,
|
||||
tmp_path / "worker",
|
||||
StreamMailbox(),
|
||||
threading.Event(),
|
||||
readiness=next_monitor,
|
||||
clock_ns=lambda: clock[0],
|
||||
)
|
||||
try:
|
||||
next_run.ready()
|
||||
with pytest.raises(WorkerLeaseError, match="mismatch"):
|
||||
next_run.validate_result_binding(run.start.to_dict())
|
||||
assert next_run.state == GraphState.RUNNING
|
||||
finally:
|
||||
assert next_run.close()
|
||||
Reference in New Issue
Block a user