feat(perception): add bounded binary stream ingress
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,312 @@
|
||||
"""CPU-only, bounded IPC evidence on Worker; NOT a model/network qualification.
|
||||
|
||||
Recorded mode transfers original fMP4 init/media plus existing normalized
|
||||
point/pose increments at original 1x release times. Receiver is never told the
|
||||
source length. The consumer only hashes raw input, it does not decode/infer.
|
||||
Truncated mode is explicitly synthetic and exercises multi-fragment failure.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import resource
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from pilot_source import SensorArchive, camera_events, merged_events
|
||||
|
||||
from k1link.compute.live_perception import LiveIngressEvent
|
||||
from k1link.perception import streaming_wire as wire
|
||||
from k1link.perception.realtime_contract import StreamStart
|
||||
from k1link.perception.streaming_ingress import StreamingIngress
|
||||
from k1link.perception.streaming_lifecycle import StreamingLifecycle
|
||||
from k1link.perception.streaming_queue import StreamMailbox
|
||||
from k1link.perception.streaming_sender import StreamingSender
|
||||
|
||||
|
||||
def record(event):
|
||||
return {
|
||||
**wire.event_metadata(event),
|
||||
"payload_bytes": len(event.payload),
|
||||
"payload_sha256": hashlib.sha256(event.payload).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def emit_json(stream, value):
|
||||
stream.write(json.dumps(value, sort_keys=True) + "\n")
|
||||
stream.flush()
|
||||
|
||||
|
||||
def bounded_file(path, maximum):
|
||||
with path.open("rb") as stream:
|
||||
raw = stream.read(maximum + 1)
|
||||
if not 0 < len(raw) <= maximum:
|
||||
raise ValueError("source member exceeds bound")
|
||||
return raw
|
||||
|
||||
|
||||
def recorded_events(root):
|
||||
camera_root = Path("/camera")
|
||||
camera_index = camera_root / "index.jsonl"
|
||||
first = next(camera_events(camera_index, 1)).time_ns
|
||||
zero = first - 500_000_000
|
||||
wall = time.monotonic_ns() + 50_000_000
|
||||
seq = 1
|
||||
yield (
|
||||
LiveIngressEvent(
|
||||
seq,
|
||||
"recorded-acquisition",
|
||||
1,
|
||||
"camera-init",
|
||||
"sensor.camera.right",
|
||||
0,
|
||||
0,
|
||||
zero,
|
||||
bounded_file(camera_root / "init.mp4", wire.MAX_FRAGMENT),
|
||||
),
|
||||
wall,
|
||||
)
|
||||
archive = SensorArchive(Path("/sensor-source.npz"))
|
||||
try:
|
||||
for observation in merged_events(archive, camera_index, 32):
|
||||
if observation.time_ns < zero:
|
||||
continue
|
||||
due = wall + observation.time_ns - zero
|
||||
threading.Event().wait(max(0, (due - time.monotonic_ns()) / 1e9))
|
||||
seq += 1
|
||||
utc = 0 # The normalized sensor archive has no UTC evidence; do not invent it.
|
||||
if observation.channel == "camera":
|
||||
row = observation.value
|
||||
path = (camera_root / row["path"]).resolve()
|
||||
if not path.is_relative_to(camera_root):
|
||||
raise ValueError("camera member escaped source root")
|
||||
raw = bounded_file(path, wire.MAX_FRAGMENT)
|
||||
if len(raw) != row["length"] or hashlib.sha256(raw).hexdigest() != row["sha256"]:
|
||||
raise ValueError("camera chunk integrity failed")
|
||||
modality, source_id, utc = (
|
||||
"camera-frame",
|
||||
"sensor.camera.right",
|
||||
row["host_epoch_ns"],
|
||||
)
|
||||
elif observation.channel == "points":
|
||||
xyz, intensity = observation.value
|
||||
raw = struct.pack("<I", len(xyz)) + xyz.tobytes() + intensity.tobytes()
|
||||
modality, source_id = "lidar", "normalized-map-point-increments"
|
||||
else:
|
||||
position, quaternion = observation.value
|
||||
raw = position.tobytes() + quaternion.tobytes()
|
||||
modality, source_id = "pose", "normalized-map-from-lidar-pose"
|
||||
yield (
|
||||
LiveIngressEvent(
|
||||
seq,
|
||||
"recorded-acquisition",
|
||||
1,
|
||||
modality,
|
||||
source_id,
|
||||
observation.sequence,
|
||||
utc,
|
||||
observation.time_ns,
|
||||
raw,
|
||||
),
|
||||
due,
|
||||
)
|
||||
if observation.channel == "camera" and observation.sequence == 31:
|
||||
break
|
||||
finally:
|
||||
(root / "source-reads.json").write_text(json.dumps(archive.counters(), indent=2))
|
||||
archive.close()
|
||||
|
||||
|
||||
def producer(args):
|
||||
root = Path(args.output)
|
||||
identity = StreamStart.from_dict(json.loads((root / "start.json").read_text()))
|
||||
connection = socket.socket(fileno=args.fd)
|
||||
if connection.recv(1) != b"R":
|
||||
raise ValueError("runtime did not become ready")
|
||||
with (root / "sent.jsonl").open("w") as journal:
|
||||
if args.case == "recorded":
|
||||
sender = StreamingSender(connection, identity, "recorded-acquisition", 1, lambda: None)
|
||||
for event, due in recorded_events(root):
|
||||
sent = time.monotonic_ns()
|
||||
sender.send(event)
|
||||
emit_json(
|
||||
journal,
|
||||
{
|
||||
**record(event),
|
||||
"due_monotonic_ns": str(due),
|
||||
"send_start_monotonic_ns": str(sent),
|
||||
"send_end_monotonic_ns": str(time.monotonic_ns()),
|
||||
},
|
||||
)
|
||||
end_ns = time.monotonic_ns()
|
||||
sender.end()
|
||||
else:
|
||||
connection.sendall(wire.open_packet(identity, "recorded-acquisition", 1))
|
||||
event = LiveIngressEvent(
|
||||
1,
|
||||
"recorded-acquisition",
|
||||
1,
|
||||
"lidar",
|
||||
"synthetic",
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
b"x" * (wire.MAX_FRAGMENT + 17),
|
||||
)
|
||||
header, payload = next(wire.event_packets(identity, event))
|
||||
connection.sendall(header)
|
||||
connection.sendall(payload)
|
||||
emit_json(journal, record(event))
|
||||
end_ns = time.monotonic_ns()
|
||||
connection.close() # Deliberate EOF between fragments; no End.
|
||||
(root / "source-end.json").write_text(json.dumps({"end_monotonic_ns": str(end_ns)}))
|
||||
# The supervised source child remains alive until owned-group cleanup.
|
||||
# Its normal completion is not mistaken for an unexpected model death.
|
||||
threading.Event().wait(30)
|
||||
|
||||
|
||||
def run_case(base, case, generation):
|
||||
root = base / case
|
||||
root.mkdir()
|
||||
specification = {
|
||||
"case": case,
|
||||
"transport": wire.SCHEMA,
|
||||
"consumer": "raw-digest-only",
|
||||
"camera": "original-fmp4-init-and-media",
|
||||
"points": "u32le-count+xyz-f64le+intensity-u8",
|
||||
"pose": "position-f64le3+map-from-lidar-quaternion-f64le4",
|
||||
"fragment_timeout_ms": 250,
|
||||
"idle_timeout_ms": 2000,
|
||||
}
|
||||
identity = StreamStart(
|
||||
run_id=f"binary-ingress-{case}",
|
||||
source_id="RAVNOVES00-prefix" if case == "recorded" else "synthetic",
|
||||
worker_id="worker-006",
|
||||
epoch_id=f"ipc-{generation}",
|
||||
lease_generation=generation,
|
||||
profile_sha256=hashlib.sha256(
|
||||
bounded_file(base / "candidate-profile.json", 65536)
|
||||
).hexdigest(),
|
||||
image_sha256="664824aa25de1db178f177d67a81b01541a938b479812b9383ddbf03f6b59dbe",
|
||||
effective_config_sha256=hashlib.sha256(wire.canonical(specification)).hexdigest(),
|
||||
# Opaque passthrough doesn't resolve or use calibration. Explicit synthetic
|
||||
# identity is NOT eligible for a real perception scene.
|
||||
calibration_sha256=hashlib.sha256(b"opaque-transport-no-calibration-use").hexdigest(),
|
||||
clock_domain_id="original-host-arrival-clock",
|
||||
input_mode="recorded-source-paced",
|
||||
)
|
||||
(root / "start.json").write_text(json.dumps(identity.to_dict(), indent=2))
|
||||
run = StreamingLifecycle(
|
||||
identity, Path("/tmp/worker-lease"), StreamMailbox(), threading.Event()
|
||||
)
|
||||
heartbeat_stop = threading.Event()
|
||||
|
||||
def renew():
|
||||
while not heartbeat_stop.wait(0.25):
|
||||
run.renew(identity)
|
||||
|
||||
heartbeat = threading.Thread(target=renew, daemon=True)
|
||||
heartbeat.start()
|
||||
left, right = socket.socketpair()
|
||||
receiver = None
|
||||
started = time.monotonic_ns()
|
||||
with (root / "producer.log").open("wb") as log, (root / "received.jsonl").open("w") as journal:
|
||||
try:
|
||||
run.spawn(
|
||||
lambda: subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-B",
|
||||
__file__,
|
||||
"--producer",
|
||||
"--case",
|
||||
case,
|
||||
"--fd",
|
||||
str(left.fileno()),
|
||||
"--output",
|
||||
str(root),
|
||||
],
|
||||
pass_fds=(left.fileno(),),
|
||||
start_new_session=True,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=log,
|
||||
stderr=log,
|
||||
)
|
||||
)
|
||||
left.close()
|
||||
|
||||
def consume(event):
|
||||
emit_json(
|
||||
journal,
|
||||
{
|
||||
**record(event),
|
||||
"received_monotonic_ns": str(time.monotonic_ns()),
|
||||
"original_received_monotonic_ns": str(event.received_monotonic_ns),
|
||||
},
|
||||
)
|
||||
|
||||
receiver = StreamingIngress(
|
||||
right,
|
||||
run,
|
||||
"recorded-acquisition",
|
||||
1,
|
||||
consume,
|
||||
lambda notice: emit_json(journal, {"notice": notice}),
|
||||
)
|
||||
run.ready()
|
||||
right.sendall(b"R")
|
||||
receiver.start()
|
||||
if not receiver.join(timeout=15):
|
||||
raise TimeoutError("bounded stream probe did not finish")
|
||||
finally:
|
||||
run.request_stop("completed" if receiver and receiver.terminal == "end" else "failed")
|
||||
run.stop_children()
|
||||
left.close()
|
||||
right.close()
|
||||
if receiver:
|
||||
receiver.join()
|
||||
heartbeat_stop.set()
|
||||
heartbeat.join(timeout=1)
|
||||
released = run.close()
|
||||
result = {
|
||||
"created_utc": datetime.now(UTC).isoformat(),
|
||||
"started_monotonic_ns": str(started),
|
||||
"finished_monotonic_ns": str(time.monotonic_ns()),
|
||||
"specification": specification,
|
||||
"receiver": receiver.snapshot(),
|
||||
"lifecycle": run.snapshot(),
|
||||
"released": released,
|
||||
"peak_input_bytes": run.mailbox.peak_bytes,
|
||||
"residual_input_bytes": run.mailbox.bytes,
|
||||
"maximum_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
|
||||
"child_maximum_rss_kib": resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss,
|
||||
"cgroup_memory_peak": Path("/sys/fs/cgroup/memory.peak").read_text().strip(),
|
||||
"model_runs": 0,
|
||||
"network_qualified": False,
|
||||
"actuation_allowed": False,
|
||||
}
|
||||
(root / "report.json").write_text(json.dumps(result, indent=2))
|
||||
assert released and run.mailbox.bytes == 0
|
||||
assert receiver.terminal == ("end" if case == "recorded" else "failed")
|
||||
assert receiver.snapshot()["incomplete_observations"] == (0 if case == "recorded" else 1)
|
||||
print(json.dumps(result), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--producer", action="store_true")
|
||||
parser.add_argument("--fd", type=int)
|
||||
parser.add_argument("--case", choices=("recorded", "truncated"))
|
||||
parser.add_argument("--output", default="/out")
|
||||
args = parser.parse_args()
|
||||
if args.producer:
|
||||
producer(args)
|
||||
else:
|
||||
for generation, case in enumerate(("recorded", "truncated"), 1):
|
||||
run_case(Path(args.output), case, generation)
|
||||
Reference in New Issue
Block a user