SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): trace M4.8S pipeline latency

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-25 18:21:57 +03:00
parent 5b07372791
commit a0152f371e
7 changed files with 453 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/perception/run_m48s_reference_graph_shadow_worker.py
+208 -5
View File
@@ -19,9 +19,9 @@ from typing import Any, Final, TextIO, cast
import numpy as np
from k1link.perception.contracts import MotionState
from k1link.perception.contracts import ObjectProposal2D
from k1link.perception.contracts import MotionState, ObjectProposal2D
from k1link.perception.detector import (
DetectorFrameTiming,
DetectorProviderSnapshot,
RfDetrShadowDetectorProvider,
)
@@ -44,13 +44,15 @@ from k1link.perception.m48s_reference_graph_runtime import (
)
from k1link.perception.motion import ClassIndependentMotionEstimator
from k1link.perception.object_understanding import AdvisoryResponse
from k1link.perception.reference_graph_runtime import ReferenceGraphRuntimePaths
from k1link.perception.providers import SourcePacket
from k1link.perception.recorded_source import DecodedFrameTiming
from k1link.perception.reference_graph_runtime import ReferenceGraphRuntimePaths
from k1link.perception.rolling_map import RollingLocalObstacleMapProvider
from k1link.perception.temporal import BoundedSpatialTemporalProvider
SCHEMA_VERSION: Final = "missioncore.m48s-reference-graph-shadow-load/v0"
FRAME_EVIDENCE_SCHEMA: Final = "missioncore.m48s-reference-graph-frame-evidence/v0"
SCHEMA_VERSION: Final = "missioncore.m48s-reference-graph-shadow-load/v1"
FRAME_EVIDENCE_SCHEMA: Final = "missioncore.m48s-reference-graph-frame-evidence/v1"
PIPELINE_TIMING_SCHEMA: Final = "missioncore.m48s-frame-pipeline-timing/v0"
AUTHORITY: Final = {
"ground_truth": False,
"candidate_accepted": False,
@@ -104,6 +106,113 @@ class GpuTelemetry:
self._stop.wait(self.interval_seconds)
class FrameTimingStore:
"""Join bounded decode, detector and provider timings by source sequence."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._decode: dict[int, int] = {}
self._detector: dict[int, DetectorFrameTiming] = {}
self._providers: dict[int, dict[str, int]] = defaultdict(dict)
self._delivered: list[dict[str, object]] = []
def observe_decode(self, timing: DecodedFrameTiming) -> None:
with self._lock:
self._decode[timing.sequence] = timing.duration_ns
def observe_detector(self, timing: DetectorFrameTiming) -> None:
with self._lock:
self._detector[timing.sequence] = timing
def observe_provider(self, stage_id: str, sequence: int, duration_ns: int) -> None:
with self._lock:
stages = self._providers[sequence]
if stage_id in stages:
raise RuntimeError("provider timing stage was recorded twice")
stages[stage_id] = max(0, duration_ns)
def take(
self,
*,
sequence: int,
source_age_ns: int,
completion_age_ns: int,
) -> dict[str, object]:
with self._lock:
try:
decode_ns = self._decode.pop(sequence)
detector = self._detector.pop(sequence)
providers = self._providers.pop(sequence)
except KeyError as exc:
raise RuntimeError("delivered frame pipeline timing is incomplete") from exc
expected_stages = {"geometry", "temporal", "motion", "rolling", "threat"}
if set(providers) != expected_stages:
raise RuntimeError("delivered frame provider timing stages are incomplete")
admission_to_delivery_ns = max(0, completion_age_ns - source_age_ns)
provider_ns = sum(providers.values())
attributed_graph_ns = detector.total_duration_ns + provider_ns
unattributed_ns = max(0, admission_to_delivery_ns - attributed_graph_ns)
document = {
"schema_version": PIPELINE_TIMING_SCHEMA,
"sequence": sequence,
"decode_duration_ns": decode_ns,
"detector": detector.to_dict(),
"providers": dict(sorted(providers.items())),
"graph_admission_to_delivery_ns": admission_to_delivery_ns,
"graph_attributed_provider_ns": attributed_graph_ns,
"graph_unattributed_ns": unattributed_ns,
"decode_to_delivery_processing_ns": decode_ns + admission_to_delivery_ns,
}
with self._lock:
self._delivered.append(document)
return document
def delivered(self) -> tuple[dict[str, object], ...]:
with self._lock:
return tuple(self._delivered)
class TimedProviderProxy:
"""Record one provider's actual call duration without another inference pass."""
def __init__(self, stage_id: str, provider: object, store: FrameTimingStore) -> None:
self.stage_id = stage_id
self.provider = provider
self.store = store
self.provider_id = cast(Any, provider).provider_id
def _call(self, sequence: int, method: str, *args: object) -> object:
started_ns = time.perf_counter_ns()
try:
return getattr(self.provider, method)(*args)
finally:
self.store.observe_provider(
self.stage_id,
sequence,
max(0, time.perf_counter_ns() - started_ns),
)
def associate(self, packet: SourcePacket, proposals: object) -> object:
return self._call(packet.envelope.sequence, "associate", packet, proposals)
def update(self, packet: SourcePacket, values: object) -> object:
return self._call(packet.envelope.sequence, "update", packet, values)
def estimate(self, packet: SourcePacket, obstacles: object) -> object:
return self._call(packet.envelope.sequence, "estimate", packet, obstacles)
def assess(self, obstacle_map: object) -> object:
frame_id = cast(Any, obstacle_map).frame_id
try:
sequence = int(str(frame_id).rsplit("-", 1)[1])
except (IndexError, ValueError) as exc:
raise RuntimeError("threat timing frame id is incompatible") from exc
return self._call(sequence, "assess", obstacle_map)
def snapshot(self) -> object:
return cast(Any, self.provider).snapshot()
def main() -> int:
parser = argparse.ArgumentParser()
for name in (
@@ -161,6 +270,7 @@ def main() -> int:
completion_ages_ms: list[float] = []
map_output_ages_ms: list[float] = []
all_deliveries: list[DeliveredFrame] = []
all_pipeline_timings: list[dict[str, object]] = []
with (
progress.open("x", encoding="utf-8") as progress_stream,
frame_ledger.open("x", encoding="utf-8") as frame_ledger_stream,
@@ -168,6 +278,7 @@ def main() -> int:
):
for loop_index in range(arguments.loops):
loop_completion_ages_ns: list[int] = []
timing_store = FrameTimingStore()
setup_started_ns = time.monotonic_ns()
with build_m48s_reference_graph_runtime(
@@ -183,9 +294,28 @@ def main() -> int:
_record_delivery_evidence,
frame_ledger_stream,
loop_index,
timing_store,
),
decode_timing_observer=timing_store.observe_decode,
detector_timing_observer=timing_store.observe_detector,
maximum_frames=arguments.maximum_frames,
) as runtime:
for stage_id, attribute in (
("geometry", "geometry"),
("temporal", "temporal"),
("motion", "motion"),
("rolling", "rolling"),
("threat", "threat"),
):
setattr(
runtime.graph,
attribute,
TimedProviderProxy(
stage_id,
getattr(runtime.graph, attribute),
timing_store,
),
)
loop_started_ns = time.monotonic_ns()
result = runtime.graph.run()
loop_completed_ns = time.monotonic_ns()
@@ -225,6 +355,9 @@ def main() -> int:
setup_seconds = (loop_started_ns - setup_started_ns) / 1_000_000_000.0
if len(loop_completion_ages_ns) != len(result.deliveries):
raise RuntimeError("final delivery timing accounting did not close")
loop_pipeline_timings = timing_store.delivered()
if len(loop_pipeline_timings) != len(result.deliveries):
raise RuntimeError("pipeline timing accounting did not close")
loop_document = _loop_document(
loop_index=loop_index,
result=result,
@@ -241,6 +374,7 @@ def main() -> int:
for delivery in result.deliveries
)
all_deliveries.extend(result.deliveries)
all_pipeline_timings.extend(loop_pipeline_timings)
frame_ledger_stream.flush()
progress_row = {
"loop": loop_index + 1,
@@ -313,6 +447,7 @@ def main() -> int:
"conservative_unknown_motion_advisory": _unknown_motion_is_conservative(advisories),
"distinct_class_family_policy": len(set(advisory_policy_matrix().values()))
== len(AdvisoryFamily),
"complete_pipeline_timing": len(all_pipeline_timings) == delivered,
"authority_remains_false": all(value is False for value in AUTHORITY.values()),
}
integrated_runtime_gate_passed = all(checks.values())
@@ -351,6 +486,7 @@ def main() -> int:
"local_obstacle_map_output_age_ms": _distribution(map_output_ages_ms),
"identity_continuity": identity,
"semantic_advisory": semantic,
"pipeline_timing": _pipeline_timing_metrics(all_pipeline_timings),
"gpu": _telemetry_summary(gpu.samples),
"process_peak_rss_before_mib": round(rss_before_kib / 1024.0, 6),
"process_peak_rss_after_mib": round(rss_after_kib / 1024.0, 6),
@@ -382,6 +518,7 @@ def _record_completion_age(
def _record_delivery_evidence(
stream: TextIO,
loop_index: int,
timing_store: FrameTimingStore,
delivery: DeliveredFrame,
packet: SourcePacket,
proposals: tuple[ObjectProposal2D, ...],
@@ -396,6 +533,11 @@ def _record_delivery_evidence(
"source_envelope": packet.envelope.to_dict(),
"completion_age_ns": completion_age_ns,
"local_obstacle_map_output_age_ns": delivery.obstacle_map.output_age_ns,
"pipeline_timing": timing_store.take(
sequence=delivery.sequence,
source_age_ns=packet.envelope.source_age_ns,
completion_age_ns=completion_age_ns,
),
"delivery": delivery.canonical_dict(),
"detector_proposals": [proposal.to_dict() for proposal in proposals],
"associated_proposal_ids": sorted(associated_proposal_ids),
@@ -536,6 +678,67 @@ def _distribution(values: list[float]) -> dict[str, float]:
}
def _pipeline_timing_metrics(
documents: list[dict[str, object]],
) -> dict[str, object]:
detector_fields = (
"preprocess_duration_ns",
"inference_transport_duration_ns",
"postprocess_duration_ns",
"total_duration_ns",
)
provider_fields = ("geometry", "temporal", "motion", "rolling", "threat")
detector_values: dict[str, list[float]] = {key: [] for key in detector_fields}
provider_values: dict[str, list[float]] = {key: [] for key in provider_fields}
top_level_fields = (
"decode_duration_ns",
"graph_admission_to_delivery_ns",
"graph_attributed_provider_ns",
"graph_unattributed_ns",
"decode_to_delivery_processing_ns",
)
top_level_values: dict[str, list[float]] = {key: [] for key in top_level_fields}
for document in documents:
detector = cast(Mapping[str, int], document["detector"])
providers = cast(Mapping[str, int], document["providers"])
for key in detector_fields:
detector_values[key].append(detector[key] / 1_000_000.0)
for key in provider_fields:
provider_values[key].append(providers[key] / 1_000_000.0)
for key in top_level_fields:
top_level_values[key].append(cast(int, document[key]) / 1_000_000.0)
maximum = max(
documents,
key=lambda document: cast(int, document["graph_admission_to_delivery_ns"]),
default=None,
)
return {
"sample_count": len(documents),
"decode_duration_ms": _distribution(top_level_values["decode_duration_ns"]),
"detector_ms": {
key.removesuffix("_duration_ns"): _distribution(values)
for key, values in detector_values.items()
},
"provider_ms": {
key: _distribution(values) for key, values in provider_values.items()
},
"graph_admission_to_delivery_ms": _distribution(
top_level_values["graph_admission_to_delivery_ns"]
),
"graph_attributed_provider_ms": _distribution(
top_level_values["graph_attributed_provider_ns"]
),
"graph_unattributed_ms": _distribution(top_level_values["graph_unattributed_ns"]),
"decode_to_delivery_processing_ms": _distribution(
top_level_values["decode_to_delivery_processing_ns"]
),
"maximum_graph_sequence": (
cast(int, maximum["sequence"]) if maximum is not None else None
),
"additional_inference_passes": 0,
}
def _telemetry_summary(samples: list[dict[str, float]]) -> dict[str, Any]:
result: dict[str, Any] = {"sample_count": len(samples)}
for key in (