SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): preserve resident runtime across recoverable input gaps

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-09-02 15:27:06 +03:00
parent 06a66a3da8
commit c31c46caa5
14 changed files with 1041 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_perception_binary_bridge.py
+6 -1
View File
@@ -229,7 +229,12 @@ def test_bridge_initialization_failure_releases_unstarted_resources(monkeypatch,
monkeypatch.setattr(bridge.socket, "socketpair", sockets)
# Only constructor ownership is under test, without a live lease/thread.
monkeypatch.setattr("k1link.perception.streaming_ingress.wire.binding", lambda start: {})
runtime = SimpleNamespace(mailbox=mailbox, start=None, track_thread=register)
runtime = SimpleNamespace(
mailbox=mailbox,
start=None,
track_thread=register,
register_ingress=lambda thread, epoch: register(thread),
)
source = SimpleNamespace(source_zero=0, wall_zero=0, run=lambda sock: None)
try:
with pytest.raises(ValueError, match="injected"):
tests/test_perception_streaming_continuity.py
+353
View File
@@ -0,0 +1,353 @@
"""Small synthetic pause/resume tests: never model/GPU load on the Mac."""
import socket
import threading
from dataclasses import replace
import pytest
from k1link.compute.live_perception import LiveIngressEvent
from k1link.perception import streaming_wire as wire
from k1link.perception.graph_contracts import GraphState
from k1link.perception.realtime_contract import StreamStart
from k1link.perception.streaming_continuity import ResumeEvidence, StreamSuspended
from k1link.perception.streaming_ingress import StreamingIngress
from k1link.perception.streaming_lifecycle import StreamingLifecycle
from k1link.perception.streaming_queue import StreamMailbox
from k1link.perception.streaming_scheduler import SerialGpuStage
from k1link.perception.worker_lease import WorkerLeaseError
from k1link.perception.worker_operating_envelope import WorkerOperatingEnvelope, WorkerSnapshot
from k1link.perception.worker_readiness import WorkerReadinessMonitor
def binding():
return StreamStart(
"run",
"source",
"worker",
"epoch",
1,
"a" * 64,
"b" * 64,
"c" * 64,
"d" * 64,
"source-clock",
"live",
)
def snapshot(now, **changes):
return replace(
WorkerSnapshot(
"worker",
"worker-clock",
now,
"GPU",
"driver",
"b" * 64,
"c" * 64,
8000,
8192,
2610,
10251,
"run",
1,
(),
True,
),
**changes,
)
@pytest.fixture
def harness(tmp_path):
runs = []
def create(*, telemetry=False):
clock = [1_000_000_000]
monitor = (
WorkerReadinessMonitor(
binding(),
WorkerOperatingEnvelope("envelope", "GPU", "driver", 8000, 8192, 2610, 10251),
snapshot(clock[0]),
mode="strict-envelope",
clock_domain_id="worker-clock",
now_monotonic_ns=clock[0],
recoverable=True,
)
if telemetry
else None
)
run = StreamingLifecycle(
binding(),
tmp_path / "lease",
StreamMailbox(),
threading.Event(),
clock_ns=lambda: clock[0],
recover_input=True,
source_clock_ns=lambda: clock[0],
readiness=monitor,
)
run.ready()
runs.append(run)
return run, clock
yield create
for run in runs:
assert run.close()
def proof(now):
return ResumeEvidence(now, now, now, now, True)
def test_pause_keeps_lease_mailbox_and_warm_models_then_new_epoch(harness):
run, clock = harness()
run.admit(run.start, {"sequence": 100, "payload_bytes": 8})
run.pause_input(run.start, "input-disconnected")
assert run.state == GraphState.RUNNING and not run.stop_event.is_set()
assert not run.mailbox.done and not run.mailbox.bytes and not run.lease.released
assert run.mailbox.drop_counts["input-gap"] == 1
for _ in range(4):
clock[0] += 900_000_000
run.renew(run.start)
with pytest.raises(StreamSuspended):
run.admit(run.start, {"sequence": 101, "payload_bytes": 8})
epoch = run.begin_input(run.start)
assert epoch.epoch_id != run.start.epoch_id and epoch.lease_generation == 1
reset = []
with pytest.raises(StreamSuspended):
run.admit(epoch, {"sequence": 0, "payload_bytes": 8})
run.resume_input(epoch, proof(clock[0]), lambda: reset.append("temporal-reset"))
assert reset == ["temporal-reset"] and run.continuity.phase == "active"
assert run.admit(epoch, {"sequence": 0, "payload_bytes": 8})
run.mailbox.release(run.mailbox.take())
with pytest.raises(StreamSuspended):
run.validate_result_binding(run.start.to_dict())
run.validate_result_binding(epoch.to_dict())
def test_active_old_callback_cannot_publish_or_be_freed_early(harness):
run, clock = harness()
packet = {"sequence": 0, "payload_bytes": 8}
run.admit(run.start, packet)
assert run.mailbox.take() is packet
with pytest.raises(StreamSuspended), run.work(run.start, "gpu"):
run.pause_input(run.start, "input-timeout")
run.check_current(run.start) # In-flight IPC can drain without poisoning the child.
with pytest.raises(StreamSuspended):
run.begin_input(run.start)
with pytest.raises(ValueError, match="still owns"):
run.begin_input(run.start)
run.mailbox.release(packet, discard_reason="input-gap")
epoch = run.begin_input(run.start)
run.resume_input(epoch, proof(clock[0]), lambda: None)
with pytest.raises(StreamSuspended):
run.pause_input(run.start, "input-timeout")
assert run.continuity.phase == "active"
@pytest.mark.parametrize(
"field,value",
[
("decoded_keyframe", False),
("pose_ns", 0),
("newest_points_ns", 0),
("oldest_points_ns", 0),
("camera_ns", 2_000_000_000),
],
)
def test_resume_requires_fresh_synchronized_evidence(harness, field, value):
run, clock = harness()
run.pause_input(run.start, "source-gap")
epoch = run.begin_input(run.start)
called = []
with pytest.raises(StreamSuspended):
run.resume_input(
epoch, replace(proof(clock[0]), **{field: value}), lambda: called.append(True)
)
assert not called and run.continuity.phase == "synchronizing" and not run.stop_event.is_set()
def test_pause_during_reset_never_reactivates_input(harness):
run, clock = harness()
run.pause_input(run.start, "source-gap")
epoch = run.begin_input(run.start)
with pytest.raises(StreamSuspended):
run.resume_input(
epoch, proof(clock[0]), lambda: run.pause_input(epoch, "input-disconnected")
)
assert run.continuity.phase == "waiting"
def test_reset_error_is_fatal_not_hidden_as_a_network_pause(harness):
run, clock = harness()
run.pause_input(run.start, "source-gap")
epoch = run.begin_input(run.start)
def broken():
raise ValueError("reset failed")
with pytest.raises(ValueError, match="reset failed"):
run.resume_input(epoch, proof(clock[0]), broken)
assert run.stop_event.is_set()
@pytest.mark.parametrize(
"changes",
[
{"competing_gpu_clients": None},
{"gpu_owner_run_id": None},
{"memory_clock_mhz": 405},
{"image_sha256": None},
],
)
def test_telemetry_unknown_or_low_clocks_pause_and_fresh_facts_require_resync(harness, changes):
run, clock = harness(telemetry=True)
clock[0] += 100_000_000
run.observe_worker(run.start, snapshot(clock[0], **changes))
assert run.continuity.phase == "waiting" and not run.stop_event.is_set()
run.renew(run.start)
with pytest.raises(StreamSuspended):
run.begin_input(run.start)
clock[0] += 100_000_000
run.observe_worker(run.start, snapshot(clock[0]))
assert run.continuity.phase == "waiting"
epoch = run.begin_input(run.start)
run.resume_input(epoch, proof(clock[0]), lambda: None)
assert run.continuity.phase == "active" and run.lease.renewals == 1
def test_expired_inventory_can_recover_but_expired_local_lease_cannot(harness):
run, clock = harness(telemetry=True)
for _ in range(4):
clock[0] += 900_000_000
run.renew(run.start)
assert run.continuity.phase == "waiting" and not run.stop_event.is_set()
run.observe_worker(run.start, snapshot(clock[0]))
epoch = run.begin_input(run.start)
run.resume_input(epoch, proof(clock[0]), lambda: None)
clock[0] += 2_000_000_000
with pytest.raises(WorkerLeaseError):
run.observe_worker(run.start, snapshot(clock[0]))
assert run.reason == "lease-lost"
@pytest.mark.parametrize(
"changes",
[
{"competing_gpu_clients": ("other-model",)},
{"gpu_owner_run_id": "other"},
{"lease_generation": 2},
{"image_sha256": "f" * 64},
],
)
def test_confirmed_owner_or_identity_conflict_still_stops(harness, changes):
run, clock = harness(telemetry=True)
clock[0] += 100_000_000
with pytest.raises(WorkerLeaseError):
run.observe_worker(run.start, snapshot(clock[0], **changes))
assert run.stop_event.is_set() and not run.lease.released
def test_serial_gpu_lane_survives_a_discarded_epoch(harness):
run, _ = harness()
seen = threading.Event()
def compute(packet):
if packet["sequence"] == 0:
seen.set()
raise StreamSuspended("discard this frame")
return packet["sequence"]
stage = SerialGpuStage(run.mailbox, compute, run.stop_event)
try:
run.admit(run.start, {"sequence": 0, "payload_bytes": 8})
assert seen.wait(1)
run.admit(run.start, {"sequence": 1, "payload_bytes": 8})
packet, result = stage.take()
assert result == 1 and stage.error is None
run.mailbox.release(packet)
finally:
assert stage.close()
@pytest.mark.parametrize("fault", ["eof", "idle", "partial", "gap"])
def test_connection_loss_keeps_runtime_and_reconnects_without_thread_growth(harness, fault):
run, clock = harness()
epoch = run.start
for index in range(10):
left, right = socket.socketpair()
seen = []
recv = StreamingIngress(
right,
run,
"capture",
1,
seen.append,
lambda _: None,
input_epoch=epoch,
idle_timeout=0.05,
fragment_timeout=0.05,
)
recv.start()
try:
left.sendall(wire.open_packet(epoch, "capture", 1))
if fault == "eof":
left.shutdown(socket.SHUT_WR)
if fault == "partial":
event = LiveIngressEvent(
1, "capture", 1, "lidar", "points", index, 0, clock[0], b"points"
)
header, _ = next(wire.event_packets(epoch, event))
left.sendall(header + b"p")
left.shutdown(socket.SHUT_WR)
if fault == "gap":
left.sendall(
wire.gap_packet(epoch, modality="camera-frame", reason="source-gap", count=1)
)
assert recv.join(1)
assert recv.terminal == "paused" and not seen and not run.stop_event.is_set()
assert (
run.continuity.phase == "waiting"
and run.mailbox.bytes == 0
and not run.mailbox.done
)
clock[0] += 1_000_000
epoch = run.begin_input(run.start)
run.resume_input(epoch, proof(clock[0]), lambda: None)
finally:
left.close()
assert len(run._threads) <= 1 and run.continuity.generation == 10
def test_old_transport_binding_and_backlog_never_reach_new_consumer(harness):
run, clock = harness()
run.pause_input(run.start, "input-timeout")
clock[0] += 100_000_000
epoch = run.begin_input(run.start)
with pytest.raises(StreamSuspended):
run.check_input(epoch, synchronizing=True, event_ns=clock[0] - 1)
with pytest.raises(StreamSuspended):
run.check_input(run.start, synchronizing=True)
assert not run.stop_event.is_set()
def test_late_old_packet_on_new_connection_does_not_kill_models(harness):
run, clock = harness()
run.pause_input(run.start, "input-disconnected")
epoch = run.begin_input(run.start)
run.resume_input(epoch, proof(clock[0]), lambda: None)
left, right = socket.socketpair()
seen = []
receiver = StreamingIngress(
right, run, "capture", 1, seen.append, lambda _: None, input_epoch=epoch
)
receiver.start()
try:
left.sendall(wire.open_packet(epoch, "capture", 1))
left.sendall(wire.terminal_packet(run.start))
assert receiver.join() and receiver.terminal == "paused"
assert not seen and not run.stop_event.is_set() and not run.mailbox.done
finally:
left.close()
tests/test_perception_streaming_decoder.py
+16
View File
@@ -162,6 +162,22 @@ def fake_av(monkeypatch):
return codec, av, opens
def test_epoch_reset_discards_predictive_state_but_keeps_decoder_object(fake_av):
decoder = decoder_module.FragmentDecoder()
init, frame = fixture()
decoder.configure(init)
decoder.decode(frame)
decoder.reset(init)
assert decoder.frames == 0 and decoder._next_dts is None
decoder.decode(fixture(dts=500)[1])
assert decoder.frames == 1 and decoder._next_dts == 600
decoder.reset(init)
with pytest.raises(decoder_module.StreamingDecodeError, match="random-access"):
decoder.decode(fixture(dts=600, sync=False)[1])
with pytest.raises(decoder_module.StreamingDecodeError, match="failed decoder"):
decoder.reset(init)
def test_persistent_codec_emits_current_frame_without_demux_or_flush(fake_av):
codec, _, opens = fake_av
init, fragment = fixture()