feat(perception): gate streaming lifecycle on controller readiness
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,206 @@
|
||||
"""CPU-only Linux lifecycle fault probe with explicitly SYNTHETIC GPU facts.
|
||||
|
||||
No models, sensor files, GPU devices or host management. This proves bounded
|
||||
controller/supervisor behavior, not actual NVML inventory collection or latency
|
||||
of the perception graph. The fixture lease is NOT the physical Worker lease.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import asdict, replace
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.perception.realtime_contract import StreamStart
|
||||
from k1link.perception.streaming_lifecycle import StreamingLifecycle
|
||||
from k1link.perception.streaming_queue import StreamMailbox
|
||||
from k1link.perception.worker_lease import WorkerLeaseError
|
||||
from k1link.perception.worker_operating_envelope import WorkerOperatingEnvelope, WorkerSnapshot
|
||||
from k1link.perception.worker_readiness import WorkerReadinessMonitor
|
||||
|
||||
|
||||
def trial(root, lease_root, generation, scenario):
|
||||
mode = "labelled-experiment" if scenario == "allowed-overload" else "strict-envelope"
|
||||
envelope = WorkerOperatingEnvelope(
|
||||
"synthetic-4090/v1", "synthetic-RTX4090", "synthetic-driver", 1000, 256, 2610, 10251
|
||||
)
|
||||
config = {"mode": mode, "envelope": asdict(envelope), "scenario": scenario}
|
||||
start = StreamStart(
|
||||
f"fixture-{generation}",
|
||||
"no-recording",
|
||||
"synthetic-worker",
|
||||
f"epoch-{generation}",
|
||||
generation,
|
||||
"a" * 64,
|
||||
"b" * 64,
|
||||
hashlib.sha256(json.dumps(config, sort_keys=True).encode()).hexdigest(),
|
||||
"d" * 64,
|
||||
"unused-source-clock",
|
||||
"live",
|
||||
)
|
||||
|
||||
def observed(**changes):
|
||||
return replace(
|
||||
WorkerSnapshot(
|
||||
start.worker_id,
|
||||
"local-worker-process",
|
||||
time.monotonic_ns(),
|
||||
envelope.gpu_name,
|
||||
envelope.driver_version,
|
||||
start.image_sha256,
|
||||
start.effective_config_sha256,
|
||||
1000,
|
||||
256,
|
||||
2610,
|
||||
10251,
|
||||
start.run_id,
|
||||
generation,
|
||||
(),
|
||||
True,
|
||||
),
|
||||
**changes,
|
||||
)
|
||||
|
||||
runtime = StreamingLifecycle(
|
||||
start,
|
||||
lease_root,
|
||||
StreamMailbox(),
|
||||
threading.Event(),
|
||||
readiness=WorkerReadinessMonitor(
|
||||
start,
|
||||
envelope,
|
||||
observed(),
|
||||
mode=mode,
|
||||
clock_domain_id="local-worker-process",
|
||||
now_monotonic_ns=time.monotonic_ns(),
|
||||
),
|
||||
)
|
||||
heartbeat_stop = threading.Event()
|
||||
|
||||
def heartbeat():
|
||||
while not heartbeat_stop.wait(0.1):
|
||||
try:
|
||||
runtime.renew(start)
|
||||
except WorkerLeaseError:
|
||||
return
|
||||
|
||||
renewer = threading.Thread(target=heartbeat, name="fixture-controller", daemon=True)
|
||||
renewer.start()
|
||||
child = None
|
||||
packet = None
|
||||
result = {"scenario": scenario, "config": config, "start_monotonic_ns": time.monotonic_ns()}
|
||||
try:
|
||||
child = runtime.spawn(
|
||||
lambda: subprocess.Popen(
|
||||
[sys.executable, "-c", "import time; print('ready',flush=True); time.sleep(15)"],
|
||||
stdout=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
)
|
||||
)
|
||||
if not select.select([child.stdout], [], [], 2)[0] or child.stdout.readline() != b"ready\n":
|
||||
raise RuntimeError("fixture child startup failed")
|
||||
runtime.ready()
|
||||
packet = {"sequence": 0, "payload_bytes": 32}
|
||||
assert runtime.admit(start, packet) and runtime.mailbox.take() is packet
|
||||
for _ in range(3):
|
||||
assert not runtime.stop_event.wait(0.1)
|
||||
runtime.observe_worker(start, observed())
|
||||
result["running_after_refresh"] = runtime.state.value
|
||||
result["fault_injected_monotonic_ns"] = time.monotonic_ns()
|
||||
if scenario != "expired-inventory":
|
||||
change = (
|
||||
{"competing_gpu_clients": None}
|
||||
if scenario == "unknown-inventory"
|
||||
else {"memory_clock_mhz": 405}
|
||||
)
|
||||
try:
|
||||
runtime.observe_worker(start, observed(**change))
|
||||
assert scenario == "allowed-overload"
|
||||
except WorkerLeaseError:
|
||||
assert scenario != "allowed-overload"
|
||||
if scenario == "allowed-overload":
|
||||
runtime.validate_result_binding(start.to_dict())
|
||||
result["overload_accepted"] = True
|
||||
runtime.observe_worker(start, observed())
|
||||
result["recovered_envelope_audit"] = runtime.snapshot()["worker_readiness"]
|
||||
runtime.request_stop("completed")
|
||||
else:
|
||||
assert runtime.stop_event.wait(2)
|
||||
assert runtime.reason == "worker-not-ready"
|
||||
try:
|
||||
runtime.observe_worker(start, observed())
|
||||
raise AssertionError("late inventory revived stopped runtime")
|
||||
except WorkerLeaseError:
|
||||
result["late_refresh_rejected"] = True
|
||||
try:
|
||||
runtime.validate_result_binding(start.to_dict())
|
||||
raise AssertionError("stopped runtime published a result")
|
||||
except WorkerLeaseError:
|
||||
result["late_result_rejected"] = True
|
||||
child.wait(timeout=5)
|
||||
result["child_exited_monotonic_ns"] = time.monotonic_ns()
|
||||
assert not runtime.close() and not runtime.lease.released
|
||||
result["active_input_prevents_retirement"] = True
|
||||
finally:
|
||||
heartbeat_stop.set()
|
||||
renewer.join(timeout=1)
|
||||
runtime.request_stop("failed")
|
||||
runtime.stop_children()
|
||||
if packet is not None:
|
||||
runtime.mailbox.release(packet)
|
||||
result["released"] = runtime.close()
|
||||
result["runtime"] = runtime.snapshot()
|
||||
result["input_bytes"] = runtime.mailbox.bytes
|
||||
result["heartbeat_alive"] = renewer.is_alive()
|
||||
result["finished_monotonic_ns"] = time.monotonic_ns()
|
||||
if child is not None:
|
||||
child.stdout.close()
|
||||
(root / f"{scenario}.json").write_text(json.dumps(result, indent=2) + "\n")
|
||||
assert result["released"] and not result["input_bytes"] and not result["heartbeat_alive"]
|
||||
stop_ns = int(result["runtime"]["stop_requested_monotonic_ns"])
|
||||
assert result["finished_monotonic_ns"] - stop_ns <= 5_000_000_000
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
root = Path(args.output)
|
||||
root.mkdir()
|
||||
report = {
|
||||
"created_utc": datetime.now(UTC).isoformat(),
|
||||
"started_monotonic_ns": time.monotonic_ns(),
|
||||
"scope": "CPU-only real Linux process lifecycle; synthetic GPU telemetry",
|
||||
"real_gpu_inventory_proved": False,
|
||||
"full_profile_executed": False,
|
||||
"gpu_devices_visible": sorted(str(p) for p in Path("/dev").glob("nvidia*")),
|
||||
"cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES"),
|
||||
}
|
||||
assert not report["gpu_devices_visible"] and report["cuda_visible_devices"] == ""
|
||||
with tempfile.TemporaryDirectory(prefix="readiness-fixture-") as directory:
|
||||
lease_root = Path(directory)
|
||||
report["trials"] = [
|
||||
trial(root, lease_root, i, case)
|
||||
for i, case in enumerate(
|
||||
("strict-low-clocks", "allowed-overload", "unknown-inventory", "expired-inventory"),
|
||||
1,
|
||||
)
|
||||
]
|
||||
report["terminal_fixture_owner"] = json.loads((lease_root / "owner.json").read_bytes())
|
||||
report["finished_monotonic_ns"] = time.monotonic_ns()
|
||||
report["passed"] = True
|
||||
(root / "report.json").write_text(json.dumps(report, indent=2) + "\n")
|
||||
print(json.dumps({"passed": True, "scenarios": len(report["trials"]), "gpu_models": 0}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user