feat(perception): connect scoped host telemetry to recoverable profile lifecycle
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,331 @@
|
||||
"""Small synthetic controller tests. No GPU, Docker, host commands or network."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.perception.realtime_contract import StreamStart
|
||||
from k1link.perception.streaming_continuity import StreamSuspended
|
||||
from k1link.perception.streaming_lifecycle import StreamingLifecycle
|
||||
from k1link.perception.streaming_queue import StreamMailbox
|
||||
from k1link.perception.worker_control import ContainerIdentityConflict, WorkerControlChannel
|
||||
from k1link.perception.worker_control_pump import WorkerControlPump
|
||||
from k1link.perception.worker_operating_envelope import WorkerOperatingEnvelope
|
||||
from k1link.perception.worker_readiness import (
|
||||
WorkerReadinessError,
|
||||
WorkerReadinessMonitor,
|
||||
WorkerTelemetryUnavailable,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def control(tmp_path):
|
||||
start = StreamStart(
|
||||
"run",
|
||||
"source",
|
||||
"worker",
|
||||
"epoch",
|
||||
1,
|
||||
*[c * 64 for c in "abcd"],
|
||||
"source-clock",
|
||||
"recorded-source-paced",
|
||||
)
|
||||
requests, responses = tmp_path / "requests", tmp_path / "responses"
|
||||
requests.mkdir()
|
||||
responses.mkdir()
|
||||
now = [1_000_000_000]
|
||||
channel = WorkerControlChannel(
|
||||
start,
|
||||
requests,
|
||||
responses,
|
||||
container_id="e" * 12,
|
||||
clock_domain_id="local-clock",
|
||||
clock_ns=lambda: now[0],
|
||||
)
|
||||
facts = {
|
||||
"worker_id": "worker",
|
||||
"container_id": "e" * 64,
|
||||
"image_sha256": "b" * 64,
|
||||
"cpu_limit_millicores": 8000,
|
||||
"memory_limit_mib": 8192,
|
||||
"gpu_name": "RTX4090",
|
||||
"driver_version": "610.47",
|
||||
"sm_clock_mhz": 2610,
|
||||
"memory_clock_mhz": 10251,
|
||||
"gpu_telemetry_available": True,
|
||||
"inventory_complete": True,
|
||||
"competing_gpu_clients": [],
|
||||
"inventory_scope": "docker-gpu-access",
|
||||
"host_process_inventory": "unproved",
|
||||
}
|
||||
|
||||
def reply(**changes):
|
||||
channel.request()
|
||||
payload = {**channel.pending, "facts": {**facts, **changes}}
|
||||
(responses / "response.json").write_text(json.dumps(payload))
|
||||
return payload
|
||||
|
||||
return channel, now, reply
|
||||
|
||||
|
||||
def poll(channel):
|
||||
return channel.poll(owner=("run", 1), warmup_complete=True)
|
||||
|
||||
|
||||
def monitor(channel, initial, now):
|
||||
return WorkerReadinessMonitor(
|
||||
channel.start,
|
||||
WorkerOperatingEnvelope(
|
||||
"test/v1",
|
||||
"RTX4090",
|
||||
"610.47",
|
||||
8000,
|
||||
8192,
|
||||
2610,
|
||||
10251,
|
||||
inventory_scope="docker-gpu-access",
|
||||
),
|
||||
initial,
|
||||
mode="labelled-experiment",
|
||||
clock_domain_id="local-clock",
|
||||
now_monotonic_ns=now[0],
|
||||
recoverable=True,
|
||||
)
|
||||
|
||||
|
||||
def test_host_reply_uses_request_time_and_local_authority_not_receipt_time(control):
|
||||
channel, now, reply = control
|
||||
payload = reply()
|
||||
now[0] += 750_000_000
|
||||
observed = poll(channel)
|
||||
assert observed.observed_monotonic_ns == 1_000_000_000
|
||||
assert observed.gpu_owner_run_id == "run" and observed.lease_generation == 1
|
||||
assert observed.effective_config_sha256 == channel.start.effective_config_sha256
|
||||
assert observed.competing_gpu_clients == ()
|
||||
assert channel.container_id == "e" * 64
|
||||
assert channel.snapshot()["last_roundtrip_ms"] == 750
|
||||
assert "observed_monotonic_ns" not in payload
|
||||
assert poll(channel) is None # A response is consumable exactly once.
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["nonce", "activation_sha256", "sequence", "extra", "oversize"])
|
||||
def test_stale_cross_activation_malformed_response_cannot_refresh_owner(control, change):
|
||||
channel, now, reply = control
|
||||
payload = reply()
|
||||
if change == "extra":
|
||||
payload["command"] = "must never run"
|
||||
elif change == "sequence":
|
||||
payload[change] += 1
|
||||
else:
|
||||
payload[change] = "f" * (17000 if change == "oversize" else 64)
|
||||
(channel.responses / "response.json").write_text(json.dumps(payload))
|
||||
assert poll(channel) is None
|
||||
now[0] += 1_000_000_001
|
||||
assert poll(channel) is None and channel.expired == 1
|
||||
channel.request()
|
||||
assert channel.sequence == 2 and channel.accepted == 0
|
||||
|
||||
|
||||
def test_delayed_reply_stays_expired_and_cannot_hide_a_known_conflict(control):
|
||||
channel, now, reply = control
|
||||
reply()
|
||||
readiness = monitor(channel, poll(channel), now)
|
||||
now[0] += 1
|
||||
reply()
|
||||
now[0] += 1_200_000_000
|
||||
with pytest.raises(WorkerTelemetryUnavailable):
|
||||
readiness.observe(
|
||||
channel.start, poll(channel), now_monotonic_ns=now[0], require_warmup=True
|
||||
)
|
||||
assert channel.expired == 1
|
||||
now[0] += 1
|
||||
reply(inventory_complete=False, competing_gpu_clients=["docker:" + "f" * 64])
|
||||
with pytest.raises(WorkerReadinessError, match="competing-gpu"):
|
||||
readiness.observe(
|
||||
channel.start, poll(channel), now_monotonic_ns=now[0], require_warmup=True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("inventory_complete", False),
|
||||
("gpu_telemetry_available", False),
|
||||
("container_id", None),
|
||||
],
|
||||
)
|
||||
def test_missing_real_facts_cannot_be_waived_by_labelled_experiment(control, field, value):
|
||||
channel, now, reply = control
|
||||
reply()
|
||||
readiness = monitor(channel, poll(channel), now)
|
||||
now[0] += 1
|
||||
reply(**{field: value})
|
||||
with pytest.raises(WorkerTelemetryUnavailable):
|
||||
readiness.observe(
|
||||
channel.start, poll(channel), now_monotonic_ns=now[0], require_warmup=True
|
||||
)
|
||||
assert not readiness.snapshot()["terminal_failures"]
|
||||
now[0] += 1
|
||||
reply()
|
||||
readiness.observe(channel.start, poll(channel), now_monotonic_ns=now[0], require_warmup=True)
|
||||
|
||||
|
||||
def test_container_id_is_pinned_and_changed_target_is_terminal(control):
|
||||
channel, _, reply = control
|
||||
reply()
|
||||
poll(channel)
|
||||
reply(container_id="e" * 12 + "f" * 52)
|
||||
with pytest.raises(ContainerIdentityConflict):
|
||||
poll(channel)
|
||||
|
||||
|
||||
def test_docker_scope_cannot_satisfy_host_wide_inventory_requirement(control):
|
||||
channel, now, reply = control
|
||||
reply()
|
||||
observed = poll(channel)
|
||||
with pytest.raises(WorkerReadinessError, match="inventory-scope-mismatch"):
|
||||
monitor(channel, replace(observed, inventory_scope="host-compute"), now)
|
||||
|
||||
|
||||
def test_real_observation_bridge_waits_resumes_but_does_not_renew_lease(control, tmp_path):
|
||||
channel, now, reply = control
|
||||
run = StreamingLifecycle(
|
||||
channel.start,
|
||||
tmp_path / "lease",
|
||||
StreamMailbox(),
|
||||
threading.Event(),
|
||||
clock_ns=lambda: now[0],
|
||||
recover_input=True,
|
||||
source_clock_ns=lambda: now[0],
|
||||
)
|
||||
try:
|
||||
reply()
|
||||
readiness = monitor(channel, poll(channel), now)
|
||||
run.attach_readiness(readiness)
|
||||
with pytest.raises(WorkerReadinessError, match="once"):
|
||||
run.attach_readiness(readiness)
|
||||
run.ready()
|
||||
now[0] += 1_000_000_001
|
||||
run.renew(channel.start)
|
||||
deadline = run.lease.deadline_ns
|
||||
with pytest.raises(StreamSuspended):
|
||||
run.check_input(channel.start)
|
||||
assert not run.stop_event.is_set()
|
||||
reply()
|
||||
run.observe_worker(channel.start, poll(channel))
|
||||
assert run.lease.deadline_ns == deadline
|
||||
assert run.continuity.phase == "waiting" # Fresh metrics aren't fresh sensor evidence.
|
||||
epoch = run.begin_input(channel.start)
|
||||
assert epoch.epoch_id != channel.start.epoch_id
|
||||
finally:
|
||||
assert run.close()
|
||||
|
||||
|
||||
def test_telemetry_lag_during_warmup_blocks_spawn_without_poisoning_first_input(control, tmp_path):
|
||||
channel, now, reply = control
|
||||
run = StreamingLifecycle(
|
||||
channel.start,
|
||||
tmp_path / "lease",
|
||||
StreamMailbox(),
|
||||
threading.Event(),
|
||||
clock_ns=lambda: now[0],
|
||||
recover_input=True,
|
||||
source_clock_ns=lambda: now[0],
|
||||
)
|
||||
spawned = []
|
||||
try:
|
||||
reply()
|
||||
run.attach_readiness(monitor(channel, poll(channel), now))
|
||||
now[0] += 1_000_000_001
|
||||
run.renew(channel.start)
|
||||
with pytest.raises(StreamSuspended):
|
||||
run.spawn(lambda: spawned.append(True))
|
||||
assert not spawned and not run.stop_event.is_set()
|
||||
assert run.continuity.phase == "active" # No input has started yet.
|
||||
reply()
|
||||
run.observe_worker(channel.start, poll(channel))
|
||||
run.ready()
|
||||
run.check_input(channel.start)
|
||||
finally:
|
||||
assert run.close()
|
||||
|
||||
|
||||
def test_controller_pump_bootstrap_refresh_ready_and_shutdown_are_independent_of_lease(
|
||||
control, tmp_path
|
||||
):
|
||||
channel, _, reply = control
|
||||
facts = reply()["facts"]
|
||||
channel.pending = None
|
||||
channel.clock_ns = time.monotonic_ns
|
||||
stopped = threading.Event()
|
||||
|
||||
def host_fixture():
|
||||
previous = None
|
||||
while not stopped.wait(0.005):
|
||||
request = json.loads((channel.requests / "request.json").read_text())
|
||||
if request["nonce"] == previous:
|
||||
continue
|
||||
path = channel.responses / "fixture.tmp"
|
||||
path.write_text(json.dumps({**request, "facts": facts}))
|
||||
os.replace(path, channel.responses / "response.json")
|
||||
previous = request["nonce"]
|
||||
|
||||
host = threading.Thread(target=host_fixture)
|
||||
host.start()
|
||||
run = StreamingLifecycle(
|
||||
channel.start,
|
||||
tmp_path / "lease",
|
||||
StreamMailbox(),
|
||||
threading.Event(),
|
||||
recover_input=True,
|
||||
source_clock_ns=time.monotonic_ns,
|
||||
)
|
||||
pump = None
|
||||
try:
|
||||
pump = WorkerControlPump(
|
||||
run,
|
||||
channel,
|
||||
WorkerOperatingEnvelope(
|
||||
"test/v1",
|
||||
"RTX4090",
|
||||
"610.47",
|
||||
8000,
|
||||
8192,
|
||||
2610,
|
||||
10251,
|
||||
inventory_scope="docker-gpu-access",
|
||||
),
|
||||
mode="labelled-experiment",
|
||||
)
|
||||
pump.ready(seconds=1)
|
||||
run.check_input(channel.start)
|
||||
assert channel.accepted >= 2 and run.lease.renewals == 0
|
||||
assert pump.snapshot()["host_process_inventory"] == "unproved"
|
||||
finally:
|
||||
if pump:
|
||||
assert pump.close()
|
||||
stopped.set()
|
||||
host.join(timeout=1)
|
||||
assert not host.is_alive() and run.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"changes",
|
||||
[
|
||||
{"inventory_complete": "true"},
|
||||
{"competing_gpu_clients": ["x"] * 65},
|
||||
{"competing_gpu_clients": "empty"},
|
||||
{"memory_limit_mib": True},
|
||||
{"host_process_inventory": "complete"},
|
||||
{"gpu_telemetry_available": "yes"},
|
||||
],
|
||||
)
|
||||
def test_schema_is_bounded_and_does_not_accept_false_host_claims(control, changes):
|
||||
channel, _, reply = control
|
||||
reply(**changes)
|
||||
assert poll(channel) is None
|
||||
assert channel.accepted == 0
|
||||
Reference in New Issue
Block a user