Добавление канонического графа M4.7
This commit is contained in:
+266
-15
@@ -29,19 +29,25 @@ from .contracts import (
|
||||
)
|
||||
from .graph_contracts import (
|
||||
GRAPH_RESULT_SCHEMA,
|
||||
GRAPH_RESULT_SCHEMA_V2,
|
||||
REFERENCE_GRAPH_ID,
|
||||
REFERENCE_GRAPH_ID_V2,
|
||||
TERMINAL_OUTCOME_SCHEMA,
|
||||
DeliveredFrame,
|
||||
GraphExecutionError,
|
||||
GraphRunMode,
|
||||
GraphRunResult,
|
||||
GraphRunResultV2,
|
||||
GraphState,
|
||||
TerminalOutcome,
|
||||
TerminalOutcomeType,
|
||||
build_graph_run_result,
|
||||
build_graph_run_result_v2,
|
||||
)
|
||||
from .graph_validation import (
|
||||
validate_observations,
|
||||
validate_proposals,
|
||||
validate_rolling,
|
||||
validate_temporal,
|
||||
validate_threats,
|
||||
)
|
||||
@@ -51,6 +57,8 @@ from .providers import (
|
||||
MotionProvider,
|
||||
ProviderRole,
|
||||
ReferencePerceptionGraphConfig,
|
||||
ReferencePerceptionGraphConfigV2,
|
||||
RollingMapProvider,
|
||||
SourcePacket,
|
||||
SourceProvider,
|
||||
TemporalStateProvider,
|
||||
@@ -79,6 +87,15 @@ class _Temporal:
|
||||
obstacles: tuple[TemporalObstacle, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Rolled:
|
||||
packet: SourcePacket
|
||||
proposals: tuple[ObjectProposal2D, ...]
|
||||
associated_proposal_ids: frozenset[str]
|
||||
obstacles: tuple[TemporalObstacle, ...]
|
||||
retained: tuple[TemporalObstacle, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _StopSignal:
|
||||
pass
|
||||
@@ -88,8 +105,9 @@ _STOP = _StopSignal()
|
||||
_DetectorItem = SourcePacket | _StopSignal
|
||||
_GeometryItem = _Detected | _StopSignal
|
||||
_TemporalItem = _Associated | _StopSignal
|
||||
_ThreatItem = _Temporal | _StopSignal
|
||||
_QueueItem = SourcePacket | _Detected | _Associated | _Temporal | _StopSignal
|
||||
_RollingItem = _Temporal | _StopSignal
|
||||
_ThreatItem = _Temporal | _Rolled | _StopSignal
|
||||
_QueueItem = SourcePacket | _Detected | _Associated | _Temporal | _Rolled | _StopSignal
|
||||
_QueueItemT = TypeVar("_QueueItemT", bound=_QueueItem)
|
||||
|
||||
|
||||
@@ -114,6 +132,37 @@ class ReferencePerceptionGraphV1:
|
||||
raise GraphExecutionError("reference graph id is incompatible")
|
||||
if config.source_profile_id != BASELINE_PROFILE_ID:
|
||||
raise GraphExecutionError("reference source profile is incompatible")
|
||||
self._initialize(
|
||||
config=config,
|
||||
source=source,
|
||||
detector=detector,
|
||||
geometry=geometry,
|
||||
temporal=temporal,
|
||||
motion=motion,
|
||||
rolling=None,
|
||||
threat=threat,
|
||||
run_mode=GraphRunMode.SOURCE_PACED_LATEST_WINS,
|
||||
telemetry_identity=telemetry_identity,
|
||||
telemetry_sink=telemetry_sink,
|
||||
clock_ns=clock_ns,
|
||||
)
|
||||
|
||||
def _initialize(
|
||||
self,
|
||||
*,
|
||||
config: ReferencePerceptionGraphConfig | ReferencePerceptionGraphConfigV2,
|
||||
source: SourceProvider,
|
||||
detector: DetectorProvider,
|
||||
geometry: GeometryAssociationProvider,
|
||||
temporal: TemporalStateProvider,
|
||||
motion: MotionProvider,
|
||||
rolling: RollingMapProvider | None,
|
||||
threat: ThreatProvider,
|
||||
run_mode: GraphRunMode,
|
||||
telemetry_identity: PipelineTelemetryIdentity | None,
|
||||
telemetry_sink: PipelineTelemetrySink | None,
|
||||
clock_ns: Callable[[], int],
|
||||
) -> None:
|
||||
if (telemetry_identity is None) is not (telemetry_sink is None):
|
||||
raise GraphExecutionError("telemetry identity and sink must be configured together")
|
||||
self.config = config
|
||||
@@ -122,7 +171,9 @@ class ReferencePerceptionGraphV1:
|
||||
self.geometry = geometry
|
||||
self.temporal = temporal
|
||||
self.motion = motion
|
||||
self.rolling = rolling
|
||||
self.threat = threat
|
||||
self.run_mode = run_mode
|
||||
self.telemetry_identity = telemetry_identity
|
||||
self.telemetry_sink = telemetry_sink
|
||||
self._clock_ns = clock_ns
|
||||
@@ -135,6 +186,9 @@ class ReferencePerceptionGraphV1:
|
||||
self._deliveries: list[DeliveredFrame] = []
|
||||
self._admitted: dict[int, SourcePacket] = {}
|
||||
self._admitted_at_ns: dict[int, int] = {}
|
||||
self._queue_high_watermarks: dict[str, int] = {
|
||||
stage_id: 0 for stage_id in ("detector", "geometry", "temporal", "rolling", "threat")
|
||||
}
|
||||
self._validate_provider_pins()
|
||||
|
||||
@property
|
||||
@@ -149,14 +203,17 @@ class ReferencePerceptionGraphV1:
|
||||
def cancel(self) -> None:
|
||||
self._cancel_event.set()
|
||||
|
||||
def run(self) -> GraphRunResult:
|
||||
def run(self) -> GraphRunResult | GraphRunResultV2:
|
||||
self._begin_run()
|
||||
started_ns = self._now()
|
||||
detector_queue: Queue[_DetectorItem] = Queue(self._capacity("detector"))
|
||||
geometry_queue: Queue[_GeometryItem] = Queue(self._capacity("geometry"))
|
||||
temporal_queue: Queue[_TemporalItem] = Queue(self._capacity("temporal"))
|
||||
threat_queue: Queue[_ThreatItem] = Queue(self._capacity("threat"))
|
||||
self._threads = [
|
||||
rolling_queue: Queue[_RollingItem] | None = (
|
||||
Queue(self._capacity("rolling")) if self.rolling is not None else None
|
||||
)
|
||||
threads = [
|
||||
Thread(
|
||||
target=self._detector_loop,
|
||||
args=(detector_queue, geometry_queue),
|
||||
@@ -171,17 +228,29 @@ class ReferencePerceptionGraphV1:
|
||||
),
|
||||
Thread(
|
||||
target=self._temporal_loop,
|
||||
args=(temporal_queue, threat_queue),
|
||||
args=(temporal_queue, rolling_queue or threat_queue),
|
||||
name="m4-temporal",
|
||||
daemon=True,
|
||||
),
|
||||
]
|
||||
if rolling_queue is not None:
|
||||
threads.append(
|
||||
Thread(
|
||||
target=self._rolling_loop,
|
||||
args=(rolling_queue, threat_queue),
|
||||
name="m4-rolling",
|
||||
daemon=True,
|
||||
)
|
||||
)
|
||||
threads.append(
|
||||
Thread(
|
||||
target=self._threat_loop,
|
||||
args=(threat_queue,),
|
||||
name="m4-threat",
|
||||
daemon=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
self._threads = threads
|
||||
for thread in self._threads:
|
||||
thread.start()
|
||||
run_emitter = self._run_emitter()
|
||||
@@ -244,6 +313,8 @@ class ReferencePerceptionGraphV1:
|
||||
self._deliveries.clear()
|
||||
self._admitted.clear()
|
||||
self._admitted_at_ns.clear()
|
||||
for stage_id in self._queue_high_watermarks:
|
||||
self._queue_high_watermarks[stage_id] = 0
|
||||
|
||||
def _admit(self, packet: SourcePacket) -> bool:
|
||||
envelope = packet.envelope
|
||||
@@ -283,6 +354,8 @@ class ReferencePerceptionGraphV1:
|
||||
validate_proposals(item, proposals)
|
||||
if stage is not None:
|
||||
stage.output_count = len(proposals)
|
||||
if self._expired(item, "detector"):
|
||||
continue
|
||||
self._put_latest(outgoing, _Detected(item, proposals), "geometry")
|
||||
except Exception as exc:
|
||||
if not isinstance(item, _StopSignal):
|
||||
@@ -311,6 +384,8 @@ class ReferencePerceptionGraphV1:
|
||||
validate_exclusive_point_ownership(observations)
|
||||
if stage is not None:
|
||||
stage.output_count = len(observations)
|
||||
if self._expired(item.packet, "geometry"):
|
||||
continue
|
||||
self._put_latest(
|
||||
outgoing,
|
||||
_Associated(item.packet, item.proposals, observations),
|
||||
@@ -325,13 +400,16 @@ class ReferencePerceptionGraphV1:
|
||||
def _temporal_loop(
|
||||
self,
|
||||
incoming: Queue[_TemporalItem],
|
||||
outgoing: Queue[_ThreatItem],
|
||||
outgoing: Queue[_RollingItem] | Queue[_ThreatItem],
|
||||
) -> None:
|
||||
while True:
|
||||
item = incoming.get()
|
||||
try:
|
||||
if isinstance(item, _StopSignal):
|
||||
self._put_stop(outgoing, "threat")
|
||||
self._put_stop(
|
||||
cast(Queue[_QueueItem], outgoing),
|
||||
"rolling" if self.rolling is not None else "threat",
|
||||
)
|
||||
return
|
||||
if self._cancelled(item.packet):
|
||||
continue
|
||||
@@ -343,8 +421,10 @@ class ReferencePerceptionGraphV1:
|
||||
validate_temporal(item.packet, obstacles)
|
||||
if stage is not None:
|
||||
stage.output_count = len(obstacles)
|
||||
if self._expired(item.packet, "temporal"):
|
||||
continue
|
||||
self._put_latest(
|
||||
outgoing,
|
||||
cast(Queue[_QueueItem], outgoing),
|
||||
_Temporal(
|
||||
item.packet,
|
||||
item.proposals,
|
||||
@@ -356,7 +436,7 @@ class ReferencePerceptionGraphV1:
|
||||
),
|
||||
obstacles,
|
||||
),
|
||||
"threat",
|
||||
"rolling" if self.rolling is not None else "threat",
|
||||
)
|
||||
except Exception as exc:
|
||||
if not isinstance(item, _StopSignal):
|
||||
@@ -364,6 +444,48 @@ class ReferencePerceptionGraphV1:
|
||||
finally:
|
||||
incoming.task_done()
|
||||
|
||||
def _rolling_loop(
|
||||
self,
|
||||
incoming: Queue[_RollingItem],
|
||||
outgoing: Queue[_ThreatItem],
|
||||
) -> None:
|
||||
rolling = self.rolling
|
||||
if rolling is None:
|
||||
raise GraphExecutionError("rolling loop requires a rolling provider")
|
||||
while True:
|
||||
item = incoming.get()
|
||||
try:
|
||||
if isinstance(item, _StopSignal):
|
||||
self._put_stop(outgoing, "threat")
|
||||
return
|
||||
if self._cancelled(item.packet):
|
||||
continue
|
||||
if self._expired(item.packet, "rolling"):
|
||||
continue
|
||||
with self._stage(item.packet, "rolling", len(item.obstacles)) as stage:
|
||||
retained = rolling.update(item.packet, item.obstacles)
|
||||
validate_rolling(item.obstacles, retained)
|
||||
if stage is not None:
|
||||
stage.output_count = len(retained)
|
||||
if self._expired(item.packet, "rolling"):
|
||||
continue
|
||||
self._put_latest(
|
||||
outgoing,
|
||||
_Rolled(
|
||||
item.packet,
|
||||
item.proposals,
|
||||
item.associated_proposal_ids,
|
||||
item.obstacles,
|
||||
retained,
|
||||
),
|
||||
"threat",
|
||||
)
|
||||
except Exception as exc:
|
||||
if not isinstance(item, _StopSignal):
|
||||
self._failed(item.packet, "rolling", exc)
|
||||
finally:
|
||||
incoming.task_done()
|
||||
|
||||
def _threat_loop(self, incoming: Queue[_ThreatItem]) -> None:
|
||||
while True:
|
||||
item = incoming.get()
|
||||
@@ -380,6 +502,8 @@ class ReferencePerceptionGraphV1:
|
||||
validate_threats(obstacle_map, threats)
|
||||
if stage is not None:
|
||||
stage.output_count = len(threats)
|
||||
if self._expired(item.packet, "threat"):
|
||||
continue
|
||||
delivery = DeliveredFrame(
|
||||
sequence=item.packet.envelope.sequence,
|
||||
obstacle_map=obstacle_map,
|
||||
@@ -399,14 +523,15 @@ class ReferencePerceptionGraphV1:
|
||||
finally:
|
||||
incoming.task_done()
|
||||
|
||||
def _obstacle_map(self, item: _Temporal) -> LocalObstacleMap:
|
||||
def _obstacle_map(self, item: _Temporal | _Rolled) -> LocalObstacleMap:
|
||||
packet = item.packet
|
||||
now_ns = self._now()
|
||||
with self._result_lock:
|
||||
admitted_at_ns = self._admitted_at_ns[packet.envelope.sequence]
|
||||
occupied = tuple(
|
||||
current = tuple(
|
||||
obstacle for obstacle in item.obstacles if obstacle.state is TemporalState.CURRENT
|
||||
)
|
||||
occupied = (*current, *item.retained) if isinstance(item, _Rolled) else current
|
||||
unknown = tuple(
|
||||
obstacle for obstacle in item.obstacles if obstacle.state is not TemporalState.CURRENT
|
||||
)
|
||||
@@ -484,9 +609,13 @@ class ReferencePerceptionGraphV1:
|
||||
item: _QueueItemT,
|
||||
stage_id: str,
|
||||
) -> None:
|
||||
if self.run_mode is GraphRunMode.LOSSLESS_REPLAY:
|
||||
self._put_lossless(queue, item, stage_id)
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
queue.put_nowait(item)
|
||||
self._record_queue_depth(stage_id, queue.qsize())
|
||||
return
|
||||
except Full:
|
||||
pass
|
||||
@@ -506,11 +635,60 @@ class ReferencePerceptionGraphV1:
|
||||
finally:
|
||||
queue.task_done()
|
||||
|
||||
def _put_lossless(
|
||||
self,
|
||||
queue: Queue[_QueueItemT],
|
||||
item: _QueueItemT,
|
||||
stage_id: str,
|
||||
) -> None:
|
||||
while True:
|
||||
try:
|
||||
queue.put(item, timeout=0.05)
|
||||
self._record_queue_depth(stage_id, queue.qsize())
|
||||
return
|
||||
except Full:
|
||||
if self._cancel_event.is_set() and not isinstance(item, _StopSignal):
|
||||
packet = _packet_from_item(item)
|
||||
self._terminal(
|
||||
packet,
|
||||
TerminalOutcomeType.FAILED,
|
||||
stage_id,
|
||||
"graph-cancelled",
|
||||
)
|
||||
return
|
||||
|
||||
def _record_queue_depth(self, stage_id: str, depth: int) -> None:
|
||||
with self._result_lock:
|
||||
self._queue_high_watermarks[stage_id] = max(
|
||||
self._queue_high_watermarks.get(stage_id, 0),
|
||||
depth,
|
||||
)
|
||||
|
||||
def _put_stop(self, queue: Queue[_QueueItemT], stage_id: str) -> None:
|
||||
timeout = self._terminal_timeout(stage_id) / 1_000_000_000
|
||||
try:
|
||||
queue.put(cast(_QueueItemT, _STOP), timeout=timeout)
|
||||
self._record_queue_depth(stage_id, queue.qsize())
|
||||
except Full:
|
||||
if self.run_mode is GraphRunMode.LOSSLESS_REPLAY:
|
||||
self._cancel_event.set()
|
||||
try:
|
||||
stranded = queue.get_nowait()
|
||||
except Empty as exc:
|
||||
raise GraphExecutionError("lossless terminal queue timed out") from exc
|
||||
try:
|
||||
if not isinstance(stranded, _StopSignal):
|
||||
self._terminal(
|
||||
_packet_from_item(stranded),
|
||||
TerminalOutcomeType.FAILED,
|
||||
stage_id,
|
||||
"terminal-queue-timeout",
|
||||
)
|
||||
finally:
|
||||
queue.task_done()
|
||||
queue.put_nowait(cast(_QueueItemT, _STOP))
|
||||
self._record_queue_depth(stage_id, queue.qsize())
|
||||
return
|
||||
self._put_latest(queue, cast(_QueueItemT, _STOP), stage_id)
|
||||
|
||||
def _join_workers(self) -> None:
|
||||
@@ -541,11 +719,27 @@ class ReferencePerceptionGraphV1:
|
||||
if set(self._admitted) != set(self._outcomes):
|
||||
raise GraphExecutionError("terminal accounting did not close")
|
||||
|
||||
def _result(self, state: GraphState) -> GraphRunResult:
|
||||
def _result(self, state: GraphState) -> GraphRunResult | GraphRunResultV2:
|
||||
with self._result_lock:
|
||||
outcomes = tuple(self._outcomes[key] for key in sorted(self._outcomes))
|
||||
deliveries = tuple(sorted(self._deliveries, key=lambda item: item.sequence))
|
||||
admitted_count = len(self._admitted)
|
||||
queue_high_watermarks = tuple(
|
||||
(stage_id, self._queue_high_watermarks[stage_id])
|
||||
for stage_id in sorted(self._queue_high_watermarks)
|
||||
if any(policy.stage_id == stage_id for policy in self.config.queues)
|
||||
)
|
||||
if self.rolling is not None:
|
||||
return build_graph_run_result_v2(
|
||||
graph_id=self.config.graph_id,
|
||||
source_profile_id=self.config.source_profile_id,
|
||||
run_mode=self.run_mode,
|
||||
state=state,
|
||||
admitted_count=admitted_count,
|
||||
outcomes=outcomes,
|
||||
deliveries=deliveries,
|
||||
queue_high_watermarks=queue_high_watermarks,
|
||||
)
|
||||
return build_graph_run_result(
|
||||
graph_id=self.config.graph_id,
|
||||
source_profile_id=self.config.source_profile_id,
|
||||
@@ -598,6 +792,8 @@ class ReferencePerceptionGraphV1:
|
||||
ProviderRole.MOTION: self.motion.provider_id,
|
||||
ProviderRole.THREAT: self.threat.provider_id,
|
||||
}
|
||||
if self.rolling is not None:
|
||||
actual[ProviderRole.ROLLING] = self.rolling.provider_id
|
||||
pinned = {pin.role: pin.provider_id for pin in self.config.providers}
|
||||
if actual != pinned:
|
||||
raise GraphExecutionError("configured provider identities do not match runtime")
|
||||
@@ -627,6 +823,56 @@ class ReferencePerceptionGraphV1:
|
||||
self._state = state
|
||||
|
||||
|
||||
class ReferencePerceptionGraphV2(ReferencePerceptionGraphV1):
|
||||
"""Final M4 graph with explicit bounded rolling occupancy ownership."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
config: ReferencePerceptionGraphConfigV2,
|
||||
source: SourceProvider,
|
||||
detector: DetectorProvider,
|
||||
geometry: GeometryAssociationProvider,
|
||||
temporal: TemporalStateProvider,
|
||||
motion: MotionProvider,
|
||||
rolling: RollingMapProvider,
|
||||
threat: ThreatProvider,
|
||||
run_mode: GraphRunMode,
|
||||
telemetry_identity: PipelineTelemetryIdentity | None = None,
|
||||
telemetry_sink: PipelineTelemetrySink | None = None,
|
||||
clock_ns: Callable[[], int] = time.monotonic_ns,
|
||||
) -> None:
|
||||
if config.graph_id != REFERENCE_GRAPH_ID_V2:
|
||||
raise GraphExecutionError("reference graph v2 id is incompatible")
|
||||
if config.source_profile_id != BASELINE_PROFILE_ID:
|
||||
raise GraphExecutionError("reference graph v2 source profile is incompatible")
|
||||
self._has_run = False
|
||||
self._run_once_lock = Lock()
|
||||
self._initialize(
|
||||
config=config,
|
||||
source=source,
|
||||
detector=detector,
|
||||
geometry=geometry,
|
||||
temporal=temporal,
|
||||
motion=motion,
|
||||
rolling=rolling,
|
||||
threat=threat,
|
||||
run_mode=run_mode,
|
||||
telemetry_identity=telemetry_identity,
|
||||
telemetry_sink=telemetry_sink,
|
||||
clock_ns=clock_ns,
|
||||
)
|
||||
|
||||
def _begin_run(self) -> None:
|
||||
with self._run_once_lock:
|
||||
if self._has_run:
|
||||
raise GraphExecutionError(
|
||||
"reference graph v2 restart requires freshly instantiated providers"
|
||||
)
|
||||
self._has_run = True
|
||||
super()._begin_run()
|
||||
|
||||
|
||||
class _NullStage:
|
||||
output_count: int | None = None
|
||||
|
||||
@@ -640,20 +886,25 @@ class _NullStage:
|
||||
def _packet_from_item(item: object) -> SourcePacket:
|
||||
if isinstance(item, SourcePacket):
|
||||
return item
|
||||
if isinstance(item, (_Detected, _Associated, _Temporal)):
|
||||
if isinstance(item, (_Detected, _Associated, _Temporal, _Rolled)):
|
||||
return item.packet
|
||||
raise GraphExecutionError("queue contained an incompatible item")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GRAPH_RESULT_SCHEMA",
|
||||
"GRAPH_RESULT_SCHEMA_V2",
|
||||
"REFERENCE_GRAPH_ID",
|
||||
"REFERENCE_GRAPH_ID_V2",
|
||||
"TERMINAL_OUTCOME_SCHEMA",
|
||||
"DeliveredFrame",
|
||||
"GraphExecutionError",
|
||||
"GraphRunResult",
|
||||
"GraphRunResultV2",
|
||||
"GraphRunMode",
|
||||
"GraphState",
|
||||
"ReferencePerceptionGraphV1",
|
||||
"ReferencePerceptionGraphV2",
|
||||
"TerminalOutcome",
|
||||
"TerminalOutcomeType",
|
||||
]
|
||||
|
||||
@@ -12,8 +12,10 @@ from typing import Final
|
||||
from .contracts import LocalObstacleMap, ThreatAssessment
|
||||
|
||||
GRAPH_RESULT_SCHEMA: Final = "missioncore.reference-perception-graph-result/v1"
|
||||
GRAPH_RESULT_SCHEMA_V2: Final = "missioncore.reference-perception-graph-result/v2"
|
||||
TERMINAL_OUTCOME_SCHEMA: Final = "missioncore.perception-terminal-outcome/v1"
|
||||
REFERENCE_GRAPH_ID: Final = "reference-perception-graph/v1"
|
||||
REFERENCE_GRAPH_ID_V2: Final = "reference-perception-graph/v2"
|
||||
_SAFE_IDENTIFIER: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,159}$")
|
||||
|
||||
|
||||
@@ -31,6 +33,13 @@ class GraphState(StrEnum):
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class GraphRunMode(StrEnum):
|
||||
"""Bounded queue behavior for realtime and deterministic replay."""
|
||||
|
||||
SOURCE_PACED_LATEST_WINS = "source-paced-latest-wins"
|
||||
LOSSLESS_REPLAY = "lossless-replay"
|
||||
|
||||
|
||||
class TerminalOutcomeType(StrEnum):
|
||||
DELIVERED = "delivered"
|
||||
SUPERSEDED = "superseded"
|
||||
@@ -110,6 +119,33 @@ class GraphRunResult:
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GraphRunResultV2:
|
||||
graph_id: str
|
||||
source_profile_id: str
|
||||
run_mode: GraphRunMode
|
||||
state: GraphState
|
||||
admitted_count: int
|
||||
terminal_outcomes: tuple[TerminalOutcome, ...]
|
||||
deliveries: tuple[DeliveredFrame, ...]
|
||||
queue_high_watermarks: tuple[tuple[str, int], ...]
|
||||
canonical_payload_sha256: str
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": GRAPH_RESULT_SCHEMA_V2,
|
||||
"graph_id": self.graph_id,
|
||||
"source_profile_id": self.source_profile_id,
|
||||
"run_mode": self.run_mode.value,
|
||||
"state": self.state.value,
|
||||
"admitted_count": self.admitted_count,
|
||||
"terminal_outcomes": [item.to_dict() for item in self.terminal_outcomes],
|
||||
"deliveries": [item.canonical_dict() for item in self.deliveries],
|
||||
"queue_high_watermarks": dict(self.queue_high_watermarks),
|
||||
"canonical_payload_sha256": self.canonical_payload_sha256,
|
||||
}
|
||||
|
||||
|
||||
def build_graph_run_result(
|
||||
*,
|
||||
graph_id: str,
|
||||
@@ -137,15 +173,52 @@ def build_graph_run_result(
|
||||
)
|
||||
|
||||
|
||||
def build_graph_run_result_v2(
|
||||
*,
|
||||
graph_id: str,
|
||||
source_profile_id: str,
|
||||
run_mode: GraphRunMode,
|
||||
state: GraphState,
|
||||
admitted_count: int,
|
||||
outcomes: tuple[TerminalOutcome, ...],
|
||||
deliveries: tuple[DeliveredFrame, ...],
|
||||
queue_high_watermarks: tuple[tuple[str, int], ...],
|
||||
) -> GraphRunResultV2:
|
||||
canonical = {
|
||||
"graph_id": graph_id,
|
||||
"source_profile_id": source_profile_id,
|
||||
"run_mode": run_mode.value,
|
||||
"terminal_outcomes": [item.to_dict() for item in outcomes],
|
||||
"deliveries": [item.canonical_dict() for item in deliveries],
|
||||
}
|
||||
payload = json.dumps(canonical, sort_keys=True, separators=(",", ":")).encode()
|
||||
return GraphRunResultV2(
|
||||
graph_id=graph_id,
|
||||
source_profile_id=source_profile_id,
|
||||
run_mode=run_mode,
|
||||
state=state,
|
||||
admitted_count=admitted_count,
|
||||
terminal_outcomes=outcomes,
|
||||
deliveries=deliveries,
|
||||
queue_high_watermarks=queue_high_watermarks,
|
||||
canonical_payload_sha256=hashlib.sha256(payload).hexdigest(),
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GRAPH_RESULT_SCHEMA",
|
||||
"GRAPH_RESULT_SCHEMA_V2",
|
||||
"REFERENCE_GRAPH_ID",
|
||||
"REFERENCE_GRAPH_ID_V2",
|
||||
"TERMINAL_OUTCOME_SCHEMA",
|
||||
"DeliveredFrame",
|
||||
"GraphExecutionError",
|
||||
"GraphRunResult",
|
||||
"GraphRunResultV2",
|
||||
"GraphRunMode",
|
||||
"GraphState",
|
||||
"TerminalOutcome",
|
||||
"TerminalOutcomeType",
|
||||
"build_graph_run_result",
|
||||
"build_graph_run_result_v2",
|
||||
]
|
||||
|
||||
@@ -69,6 +69,20 @@ def validate_temporal(
|
||||
raise GraphExecutionError("current temporal occupancy was invented without LiDAR")
|
||||
|
||||
|
||||
def validate_rolling(
|
||||
temporal: tuple[TemporalObstacle, ...],
|
||||
retained: tuple[TemporalObstacle, ...],
|
||||
) -> None:
|
||||
retained_ids = [obstacle.component_id for obstacle in retained]
|
||||
temporal_ids = {obstacle.component_id for obstacle in temporal}
|
||||
if len(set(retained_ids)) != len(retained_ids):
|
||||
raise GraphExecutionError("rolling component identities are duplicated")
|
||||
if temporal_ids.intersection(retained_ids):
|
||||
raise GraphExecutionError("rolling and temporal component identities overlap")
|
||||
if any(obstacle.state is not TemporalState.RETAINED for obstacle in retained):
|
||||
raise GraphExecutionError("rolling provider published non-retained evidence")
|
||||
|
||||
|
||||
def validate_threats(
|
||||
obstacle_map: LocalObstacleMap,
|
||||
threats: tuple[ThreatAssessment, ...],
|
||||
@@ -92,6 +106,7 @@ def validate_threats(
|
||||
__all__ = [
|
||||
"validate_observations",
|
||||
"validate_proposals",
|
||||
"validate_rolling",
|
||||
"validate_temporal",
|
||||
"validate_threats",
|
||||
]
|
||||
|
||||
@@ -19,7 +19,11 @@ from .contracts import (
|
||||
)
|
||||
|
||||
REFERENCE_GRAPH_CONFIG_SCHEMA: Final = "missioncore.reference-perception-graph-config/v1"
|
||||
REFERENCE_GRAPH_CONFIG_SCHEMA_V2: Final = "missioncore.reference-perception-graph-config/v2"
|
||||
REFERENCE_GRAPH_STAGE_IDS: Final = frozenset({"detector", "geometry", "temporal", "threat"})
|
||||
REFERENCE_GRAPH_STAGE_IDS_V2: Final = frozenset(
|
||||
{"detector", "geometry", "temporal", "rolling", "threat"}
|
||||
)
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,159}$")
|
||||
|
||||
@@ -34,9 +38,23 @@ class ProviderRole(StrEnum):
|
||||
GEOMETRY = "geometry"
|
||||
TEMPORAL = "temporal"
|
||||
MOTION = "motion"
|
||||
ROLLING = "rolling"
|
||||
THREAT = "threat"
|
||||
|
||||
|
||||
REFERENCE_GRAPH_PROVIDER_ROLES: Final = frozenset(
|
||||
{
|
||||
ProviderRole.SOURCE,
|
||||
ProviderRole.DETECTOR,
|
||||
ProviderRole.GEOMETRY,
|
||||
ProviderRole.TEMPORAL,
|
||||
ProviderRole.MOTION,
|
||||
ProviderRole.THREAT,
|
||||
}
|
||||
)
|
||||
REFERENCE_GRAPH_PROVIDER_ROLES_V2: Final = frozenset(ProviderRole)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SourcePacket:
|
||||
"""Execution-only carrier; the graph never interprets opaque sensor payloads."""
|
||||
@@ -211,7 +229,7 @@ class ReferencePerceptionGraphConfig:
|
||||
_identifier(self.graph_id, "graph id")
|
||||
_identifier(self.source_profile_id, "source profile id")
|
||||
roles = [provider.role for provider in self.providers]
|
||||
if len(set(roles)) != len(roles) or set(roles) != set(ProviderRole):
|
||||
if len(set(roles)) != len(roles) or set(roles) != REFERENCE_GRAPH_PROVIDER_ROLES:
|
||||
raise ProviderContractError("graph must pin each provider role exactly once")
|
||||
stage_ids = [queue.stage_id for queue in self.queues]
|
||||
if len(set(stage_ids)) != len(stage_ids):
|
||||
@@ -248,6 +266,57 @@ class ReferencePerceptionGraphConfig:
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReferencePerceptionGraphConfigV2:
|
||||
"""Final M4 graph contract with explicit retained rolling occupancy."""
|
||||
|
||||
graph_id: str
|
||||
source_profile_id: str
|
||||
providers: tuple[ProviderPin, ...]
|
||||
queues: tuple[QueuePolicy, ...]
|
||||
authority: GraphAuthority = GraphAuthority()
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_identifier(self.graph_id, "graph id")
|
||||
_identifier(self.source_profile_id, "source profile id")
|
||||
roles = [provider.role for provider in self.providers]
|
||||
if len(set(roles)) != len(roles) or set(roles) != REFERENCE_GRAPH_PROVIDER_ROLES_V2:
|
||||
raise ProviderContractError("graph v2 must pin each provider role exactly once")
|
||||
stage_ids = [queue.stage_id for queue in self.queues]
|
||||
if len(set(stage_ids)) != len(stage_ids):
|
||||
raise ProviderContractError("graph v2 queue policies must be unique")
|
||||
if set(stage_ids) != REFERENCE_GRAPH_STAGE_IDS_V2:
|
||||
raise ProviderContractError("graph v2 must bound each reference stage exactly once")
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": REFERENCE_GRAPH_CONFIG_SCHEMA_V2,
|
||||
"graph_id": self.graph_id,
|
||||
"source_profile_id": self.source_profile_id,
|
||||
"providers": [provider.to_dict() for provider in self.providers],
|
||||
"queues": [queue.to_dict() for queue in self.queues],
|
||||
"authority": self.authority.to_dict(),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: object) -> ReferencePerceptionGraphConfigV2:
|
||||
document = _object(value, "reference graph v2 config")
|
||||
_exact_keys(
|
||||
document,
|
||||
{"schema_version", "graph_id", "source_profile_id", "providers", "queues", "authority"},
|
||||
"reference graph v2 config",
|
||||
)
|
||||
if document.get("schema_version") != REFERENCE_GRAPH_CONFIG_SCHEMA_V2:
|
||||
raise ProviderContractError("reference graph v2 config schema is incompatible")
|
||||
return cls(
|
||||
graph_id=_string(document, "graph_id"),
|
||||
source_profile_id=_string(document, "source_profile_id"),
|
||||
providers=tuple(ProviderPin.from_dict(item) for item in _array(document, "providers")),
|
||||
queues=tuple(QueuePolicy.from_dict(item) for item in _array(document, "queues")),
|
||||
authority=GraphAuthority.from_dict(document.get("authority")),
|
||||
)
|
||||
|
||||
|
||||
class SourceProvider(Protocol):
|
||||
provider_id: str
|
||||
|
||||
@@ -290,6 +359,16 @@ class MotionProvider(Protocol):
|
||||
) -> tuple[TemporalObstacle, ...]: ...
|
||||
|
||||
|
||||
class RollingMapProvider(Protocol):
|
||||
provider_id: str
|
||||
|
||||
def update(
|
||||
self,
|
||||
packet: SourcePacket,
|
||||
obstacles: tuple[TemporalObstacle, ...],
|
||||
) -> tuple[TemporalObstacle, ...]: ...
|
||||
|
||||
|
||||
class ThreatProvider(Protocol):
|
||||
provider_id: str
|
||||
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Run and seal the M4.7 canonical graph on explicit Worker 006 inputs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from .graph_contracts import GraphRunMode, GraphRunResultV2
|
||||
from .reference_graph_parity import compare_reference_graph_to_accepted_ledgers
|
||||
from .reference_graph_result import seal_reference_graph_result
|
||||
from .reference_graph_runtime import ReferenceGraphRuntimePaths, build_reference_graph_runtime
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--graph-config", type=Path, required=True)
|
||||
parser.add_argument("--baseline-profile", type=Path, required=True)
|
||||
parser.add_argument("--geometry-profile", type=Path, required=True)
|
||||
parser.add_argument("--temporal-motion-profile", type=Path, required=True)
|
||||
parser.add_argument("--rolling-map-profile", type=Path, required=True)
|
||||
parser.add_argument("--threat-profile", type=Path, required=True)
|
||||
parser.add_argument("--camera-index", type=Path, required=True)
|
||||
parser.add_argument("--source-pack", type=Path, required=True)
|
||||
parser.add_argument("--local-surface", type=Path, required=True)
|
||||
parser.add_argument("--video", type=Path, required=True)
|
||||
parser.add_argument("--valid-fov-mask", type=Path, required=True)
|
||||
parser.add_argument("--temporal-parity-frames", type=Path, required=True)
|
||||
parser.add_argument("--threat-parity-frames", type=Path, required=True)
|
||||
parser.add_argument("--triton-origin", default="http://127.0.0.1:8000")
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=tuple(item.value for item in GraphRunMode),
|
||||
default=GraphRunMode.LOSSLESS_REPLAY.value,
|
||||
)
|
||||
parser.add_argument("--expected-frames", type=int, default=4489)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args(argv)
|
||||
paths = ReferenceGraphRuntimePaths(
|
||||
graph_config=args.graph_config,
|
||||
baseline_profile=args.baseline_profile,
|
||||
geometry_profile=args.geometry_profile,
|
||||
temporal_motion_profile=args.temporal_motion_profile,
|
||||
rolling_map_profile=args.rolling_map_profile,
|
||||
threat_profile=args.threat_profile,
|
||||
camera_index=args.camera_index,
|
||||
source_pack=args.source_pack,
|
||||
local_surface=args.local_surface,
|
||||
video=args.video,
|
||||
valid_fov_mask=args.valid_fov_mask,
|
||||
)
|
||||
started_ns = time.perf_counter_ns()
|
||||
with build_reference_graph_runtime(
|
||||
paths=paths,
|
||||
triton_origin=args.triton_origin,
|
||||
run_mode=GraphRunMode(args.mode),
|
||||
) as runtime:
|
||||
graph_result = runtime.graph.run()
|
||||
if not isinstance(graph_result, GraphRunResultV2):
|
||||
raise RuntimeError("M4.7 runtime returned a legacy graph result")
|
||||
parity = compare_reference_graph_to_accepted_ledgers(
|
||||
graph_result,
|
||||
temporal_frames_path=args.temporal_parity_frames,
|
||||
threat_frames_path=args.threat_parity_frames,
|
||||
expected_frames=args.expected_frames,
|
||||
)
|
||||
sealed = seal_reference_graph_result(
|
||||
graph_result,
|
||||
output_root=args.output_root,
|
||||
expected_frames=args.expected_frames,
|
||||
parity=parity,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"accepted": sealed.accepted,
|
||||
"elapsed_seconds": (time.perf_counter_ns() - started_ns) / 1_000_000_000,
|
||||
"result_id": sealed.result_id,
|
||||
"result_root": str(sealed.result_root),
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
)
|
||||
return 0 if sealed.accepted else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Frame-exact parity against the accepted M4.5R and M4.6 ledgers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from .contracts import TemporalState
|
||||
from .graph_contracts import GraphRunResultV2
|
||||
|
||||
REFERENCE_GRAPH_PARITY_SCHEMA: Final = "missioncore.reference-perception-graph-parity/v1"
|
||||
|
||||
|
||||
class ReferenceGraphParityError(RuntimeError):
|
||||
"""Accepted parity evidence is missing, malformed or source-inconsistent."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReferenceGraphParityReport:
|
||||
expected_frames: int
|
||||
compared_frames: int
|
||||
temporal_frames_sha256: str
|
||||
threat_frames_sha256: str
|
||||
mismatch_counts: tuple[tuple[str, int], ...]
|
||||
accepted: bool
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.expected_frames < 1 or not 0 <= self.compared_frames <= self.expected_frames:
|
||||
raise ReferenceGraphParityError("parity frame accounting is invalid")
|
||||
for digest in (self.temporal_frames_sha256, self.threat_frames_sha256):
|
||||
if len(digest) != 64 or any(
|
||||
character not in "0123456789abcdef" for character in digest
|
||||
):
|
||||
raise ReferenceGraphParityError("parity ledger digest is invalid")
|
||||
names = [name for name, _ in self.mismatch_counts]
|
||||
if (
|
||||
names != sorted(names)
|
||||
or len(set(names)) != len(names)
|
||||
or any(not name or count < 0 for name, count in self.mismatch_counts)
|
||||
):
|
||||
raise ReferenceGraphParityError("parity mismatch accounting is invalid")
|
||||
exact = self.compared_frames == self.expected_frames and all(
|
||||
count == 0 for _, count in self.mismatch_counts
|
||||
)
|
||||
if self.accepted is not exact:
|
||||
raise ReferenceGraphParityError("parity acceptance disagrees with accounting")
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": REFERENCE_GRAPH_PARITY_SCHEMA,
|
||||
"expected_frames": self.expected_frames,
|
||||
"compared_frames": self.compared_frames,
|
||||
"temporal_frames_sha256": self.temporal_frames_sha256,
|
||||
"threat_frames_sha256": self.threat_frames_sha256,
|
||||
"mismatch_counts": dict(self.mismatch_counts),
|
||||
"accepted": self.accepted,
|
||||
}
|
||||
|
||||
|
||||
def compare_reference_graph_to_accepted_ledgers(
|
||||
result: GraphRunResultV2,
|
||||
*,
|
||||
temporal_frames_path: Path,
|
||||
threat_frames_path: Path,
|
||||
expected_frames: int,
|
||||
) -> ReferenceGraphParityReport:
|
||||
if expected_frames < 1:
|
||||
raise ReferenceGraphParityError("expected parity frame count must be positive")
|
||||
if len(result.deliveries) != expected_frames:
|
||||
raise ReferenceGraphParityError("graph delivery count cannot support full parity")
|
||||
temporal_path = _regular_file(temporal_frames_path, "temporal parity ledger")
|
||||
threat_path = _regular_file(threat_frames_path, "threat parity ledger")
|
||||
mismatch = {
|
||||
"source_binding": 0,
|
||||
"current": 0,
|
||||
"rolling_retained": 0,
|
||||
"held": 0,
|
||||
"expired": 0,
|
||||
"camera_uncertainty": 0,
|
||||
"threat_assessments": 0,
|
||||
}
|
||||
compared = 0
|
||||
with temporal_path.open("rb") as temporal_stream, threat_path.open("rb") as threat_stream:
|
||||
for expected_sequence, delivery in enumerate(result.deliveries):
|
||||
temporal = _read_row(temporal_stream.readline(), "temporal", expected_sequence)
|
||||
threat = _read_row(threat_stream.readline(), "threat", expected_sequence)
|
||||
obstacle_map = delivery.obstacle_map
|
||||
if (
|
||||
delivery.sequence != expected_sequence
|
||||
or temporal.get("sequence") != expected_sequence
|
||||
or threat.get("sequence") != expected_sequence
|
||||
or temporal.get("frame_id") != obstacle_map.frame_id
|
||||
or threat.get("frame_id") != obstacle_map.frame_id
|
||||
):
|
||||
mismatch["source_binding"] += 1
|
||||
occupied_current = [
|
||||
item.to_dict()
|
||||
for item in obstacle_map.occupied
|
||||
if item.state is TemporalState.CURRENT
|
||||
]
|
||||
rolling_retained = [
|
||||
item.to_dict()
|
||||
for item in obstacle_map.occupied
|
||||
if item.state is TemporalState.RETAINED
|
||||
]
|
||||
held = [
|
||||
item.to_dict()
|
||||
for item in obstacle_map.unknown
|
||||
if item.state is TemporalState.HELD
|
||||
]
|
||||
expired = [
|
||||
item.to_dict()
|
||||
for item in obstacle_map.unknown
|
||||
if item.state is TemporalState.EXPIRED
|
||||
]
|
||||
for label, observed in (
|
||||
("current", occupied_current),
|
||||
("rolling_retained", rolling_retained),
|
||||
("held", held),
|
||||
("expired", expired),
|
||||
):
|
||||
if observed != _array(temporal, label):
|
||||
mismatch[label] += 1
|
||||
expected_camera_uncertainty = [
|
||||
_string(row, "proposal_id")
|
||||
for row in _array(threat, "camera_proposals")
|
||||
if _boolean(row, "occupied_support") is False
|
||||
]
|
||||
observed_camera_uncertainty = [
|
||||
item.proposal_id for item in obstacle_map.camera_uncertainty
|
||||
]
|
||||
if observed_camera_uncertainty != expected_camera_uncertainty:
|
||||
mismatch["camera_uncertainty"] += 1
|
||||
if [item.to_dict() for item in delivery.threats] != _array(
|
||||
threat,
|
||||
"assessments",
|
||||
):
|
||||
mismatch["threat_assessments"] += 1
|
||||
compared += 1
|
||||
if temporal_stream.readline() or threat_stream.readline():
|
||||
raise ReferenceGraphParityError("accepted parity ledger exceeds graph delivery count")
|
||||
accepted = compared == expected_frames and all(value == 0 for value in mismatch.values())
|
||||
return ReferenceGraphParityReport(
|
||||
expected_frames=expected_frames,
|
||||
compared_frames=compared,
|
||||
temporal_frames_sha256=_sha256_file(temporal_path),
|
||||
threat_frames_sha256=_sha256_file(threat_path),
|
||||
mismatch_counts=tuple(sorted(mismatch.items())),
|
||||
accepted=accepted,
|
||||
)
|
||||
|
||||
|
||||
def _regular_file(path: Path, label: str) -> Path:
|
||||
resolved = path.resolve(strict=True)
|
||||
if resolved.is_symlink() or not resolved.is_file():
|
||||
raise ReferenceGraphParityError(f"{label} must be a regular file")
|
||||
return resolved
|
||||
|
||||
|
||||
def _read_row(raw: bytes, label: str, sequence: int) -> dict[str, object]:
|
||||
if not raw:
|
||||
raise ReferenceGraphParityError(f"{label} parity ledger ended at frame {sequence}")
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ReferenceGraphParityError(f"{label} parity row is invalid JSON") from exc
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise ReferenceGraphParityError(f"{label} parity row is not an object")
|
||||
return value
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _array(document: dict[str, object], key: str) -> list[dict[str, object]]:
|
||||
value = document.get(key)
|
||||
if not isinstance(value, list) or any(
|
||||
not isinstance(item, dict) or any(not isinstance(name, str) for name in item)
|
||||
for item in value
|
||||
):
|
||||
raise ReferenceGraphParityError(f"parity {key} must be an object array")
|
||||
return value
|
||||
|
||||
|
||||
def _string(document: dict[str, object], key: str) -> str:
|
||||
value = document.get(key)
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ReferenceGraphParityError(f"parity {key} must be a nonempty string")
|
||||
return value
|
||||
|
||||
|
||||
def _boolean(document: dict[str, object], key: str) -> bool:
|
||||
value = document.get(key)
|
||||
if not isinstance(value, bool):
|
||||
raise ReferenceGraphParityError(f"parity {key} must be boolean")
|
||||
return value
|
||||
|
||||
|
||||
__all__ = [
|
||||
"REFERENCE_GRAPH_PARITY_SCHEMA",
|
||||
"ReferenceGraphParityError",
|
||||
"ReferenceGraphParityReport",
|
||||
"compare_reference_graph_to_accepted_ledgers",
|
||||
]
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Immutable result sealing for the M4.7 canonical graph shadow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from .graph_contracts import GraphRunMode, GraphRunResultV2, GraphState, TerminalOutcomeType
|
||||
from .reference_graph_parity import ReferenceGraphParityReport
|
||||
|
||||
REFERENCE_GRAPH_RESULT_PREFIX: Final = "m47-reference-graph-"
|
||||
REFERENCE_GRAPH_REPORT_SCHEMA: Final = "missioncore.reference-perception-graph-report/v1"
|
||||
REFERENCE_GRAPH_MANIFEST_SCHEMA: Final = "missioncore.reference-perception-graph-manifest/v1"
|
||||
|
||||
|
||||
class ReferenceGraphResultError(RuntimeError):
|
||||
"""A graph result cannot be sealed without complete terminal evidence."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SealedReferenceGraphResult:
|
||||
result_id: str
|
||||
result_root: Path
|
||||
accepted: bool
|
||||
report: dict[str, object]
|
||||
manifest: dict[str, object]
|
||||
|
||||
|
||||
def seal_reference_graph_result(
|
||||
result: GraphRunResultV2,
|
||||
*,
|
||||
output_root: Path,
|
||||
expected_frames: int,
|
||||
parity: ReferenceGraphParityReport,
|
||||
) -> SealedReferenceGraphResult:
|
||||
if expected_frames < 1:
|
||||
raise ReferenceGraphResultError("expected frame count must be positive")
|
||||
outcomes = Counter(item.outcome.value for item in result.terminal_outcomes)
|
||||
gates = {
|
||||
"lossless_replay_mode": result.run_mode is GraphRunMode.LOSSLESS_REPLAY,
|
||||
"graph_stopped": result.state is GraphState.STOPPED,
|
||||
"admitted_frame_count": result.admitted_count == expected_frames,
|
||||
"terminal_accounting_closed": len(result.terminal_outcomes) == expected_frames,
|
||||
"delivery_count": outcomes[TerminalOutcomeType.DELIVERED.value] == expected_frames,
|
||||
"delivery_payload_count": len(result.deliveries) == expected_frames,
|
||||
"no_failed_frames": outcomes[TerminalOutcomeType.FAILED.value] == 0,
|
||||
"no_stale_frames": outcomes[TerminalOutcomeType.STALE.value] == 0,
|
||||
"no_superseded_frames": outcomes[TerminalOutcomeType.SUPERSEDED.value] == 0,
|
||||
"no_rejected_frames": outcomes[TerminalOutcomeType.REJECTED.value] == 0,
|
||||
"no_unavailable_frames": outcomes[TerminalOutcomeType.UNAVAILABLE.value] == 0,
|
||||
"accepted_m45r_m46_parity": parity.accepted,
|
||||
}
|
||||
accepted = all(gates.values())
|
||||
report: dict[str, object] = {
|
||||
"schema_version": REFERENCE_GRAPH_REPORT_SCHEMA,
|
||||
"graph_id": result.graph_id,
|
||||
"source_profile_id": result.source_profile_id,
|
||||
"run_mode": result.run_mode.value,
|
||||
"state": result.state.value,
|
||||
"expected_frames": expected_frames,
|
||||
"admitted_frames": result.admitted_count,
|
||||
"terminal_outcomes": dict(sorted(outcomes.items())),
|
||||
"delivery_payloads": len(result.deliveries),
|
||||
"queue_high_watermarks": dict(result.queue_high_watermarks),
|
||||
"canonical_payload_sha256": result.canonical_payload_sha256,
|
||||
"accepted_parity": parity.to_dict(),
|
||||
"gates": gates,
|
||||
"accepted": accepted,
|
||||
"authority": {
|
||||
"physical_live": False,
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
root = output_root.expanduser().resolve()
|
||||
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if root.is_symlink() or not root.is_dir():
|
||||
raise ReferenceGraphResultError("result root must be a real directory")
|
||||
staging = root / f".reference-graph.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700, exist_ok=False)
|
||||
try:
|
||||
frames_path = staging / "frames.jsonl"
|
||||
outcomes_path = staging / "outcomes.jsonl"
|
||||
report_path = staging / "report.json"
|
||||
_write_json_lines(
|
||||
frames_path,
|
||||
tuple(delivery.canonical_dict() for delivery in result.deliveries),
|
||||
)
|
||||
_write_json_lines(
|
||||
outcomes_path,
|
||||
tuple(outcome.to_dict() for outcome in result.terminal_outcomes),
|
||||
)
|
||||
_write_json(report_path, report)
|
||||
file_rows = {
|
||||
name: {
|
||||
"sha256": _sha256_file(staging / name),
|
||||
"bytes": (staging / name).stat().st_size,
|
||||
}
|
||||
for name in ("frames.jsonl", "outcomes.jsonl", "report.json")
|
||||
}
|
||||
identity: dict[str, object] = {
|
||||
"graph_id": result.graph_id,
|
||||
"source_profile_id": result.source_profile_id,
|
||||
"run_mode": result.run_mode.value,
|
||||
"canonical_payload_sha256": result.canonical_payload_sha256,
|
||||
"files": file_rows,
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"{REFERENCE_GRAPH_RESULT_PREFIX}{identity_sha256}"
|
||||
manifest: dict[str, object] = {
|
||||
"schema_version": REFERENCE_GRAPH_MANIFEST_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
**identity,
|
||||
"accepted": accepted,
|
||||
}
|
||||
_write_json(staging / "manifest.json", manifest)
|
||||
target = root / result_id
|
||||
_publish_immutable(staging, target)
|
||||
except Exception:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
return SealedReferenceGraphResult(
|
||||
result_id=result_id,
|
||||
result_root=target,
|
||||
accepted=accepted,
|
||||
report=report,
|
||||
manifest=manifest,
|
||||
)
|
||||
|
||||
|
||||
def _write_json_lines(path: Path, rows: tuple[dict[str, object], ...]) -> None:
|
||||
with path.open("wb") as handle:
|
||||
for row in rows:
|
||||
handle.write(_canonical_json(row) + b"\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
|
||||
def _write_json(path: Path, document: dict[str, object]) -> None:
|
||||
with path.open("wb") as handle:
|
||||
handle.write(_canonical_json(document) + b"\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
|
||||
def _publish_immutable(staging: Path, target: Path) -> None:
|
||||
if target.exists():
|
||||
if target.is_symlink() or not target.is_dir():
|
||||
raise ReferenceGraphResultError("immutable result target is not a real directory")
|
||||
expected = {
|
||||
path.name: _sha256_file(path)
|
||||
for path in staging.iterdir()
|
||||
if path.is_file()
|
||||
}
|
||||
observed = {
|
||||
path.name: _sha256_file(path)
|
||||
for path in target.iterdir()
|
||||
if path.is_file()
|
||||
}
|
||||
if expected != observed:
|
||||
raise ReferenceGraphResultError("immutable result identity collision")
|
||||
shutil.rmtree(staging)
|
||||
return
|
||||
staging.replace(target)
|
||||
|
||||
|
||||
def _canonical_json(document: dict[str, object]) -> bytes:
|
||||
return json.dumps(document, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"REFERENCE_GRAPH_MANIFEST_SCHEMA",
|
||||
"REFERENCE_GRAPH_REPORT_SCHEMA",
|
||||
"REFERENCE_GRAPH_RESULT_PREFIX",
|
||||
"ReferenceGraphResultError",
|
||||
"SealedReferenceGraphResult",
|
||||
"seal_reference_graph_result",
|
||||
]
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Production assembly boundary for the source-neutral M4.7 reference graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .baseline import load_m4_baseline
|
||||
from .detector import FrozenYoloxDetectorProvider
|
||||
from .geometry import (
|
||||
Ravnoves00GeometryAssociationProvider,
|
||||
RecordedGeometryStore,
|
||||
load_geometry_profile,
|
||||
)
|
||||
from .graph import ReferencePerceptionGraphV2
|
||||
from .graph_contracts import GraphRunMode
|
||||
from .motion import ClassIndependentMotionEstimator
|
||||
from .providers import ProviderRole, ReferencePerceptionGraphConfigV2
|
||||
from .recorded_source import (
|
||||
DecodedRecordedSource,
|
||||
PyAvRecordedImageDecoder,
|
||||
RecordedRavnoves00Source,
|
||||
ReplayPacing,
|
||||
)
|
||||
from .rolling_map import RollingLocalObstacleMapProvider, load_rolling_map_profile
|
||||
from .temporal import BoundedSpatialTemporalProvider, load_temporal_motion_profile
|
||||
from .threat import (
|
||||
DualEvidenceReplayThreatProvider,
|
||||
RecordedReplayBodyFrameResolver,
|
||||
load_replay_threat_profile,
|
||||
)
|
||||
from .yolox_object_detector import TritonHttpInferenceBackend, load_valid_fov_mask
|
||||
|
||||
|
||||
class ReferenceGraphRuntimeError(RuntimeError):
|
||||
"""The production graph cannot be assembled from the pinned release inputs."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReferenceGraphRuntimePaths:
|
||||
graph_config: Path
|
||||
baseline_profile: Path
|
||||
geometry_profile: Path
|
||||
temporal_motion_profile: Path
|
||||
rolling_map_profile: Path
|
||||
threat_profile: Path
|
||||
camera_index: Path
|
||||
source_pack: Path
|
||||
local_surface: Path
|
||||
video: Path
|
||||
valid_fov_mask: Path
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ReferenceGraphRuntime:
|
||||
"""Own one graph and its persistent inference transport."""
|
||||
|
||||
graph: ReferencePerceptionGraphV2
|
||||
inference_backend: TritonHttpInferenceBackend
|
||||
|
||||
def close(self) -> None:
|
||||
self.inference_backend.close()
|
||||
|
||||
def __enter__(self) -> ReferenceGraphRuntime:
|
||||
return self
|
||||
|
||||
def __exit__(self, *args: object) -> None:
|
||||
self.close()
|
||||
|
||||
|
||||
def build_reference_graph_runtime(
|
||||
*,
|
||||
paths: ReferenceGraphRuntimePaths,
|
||||
triton_origin: str,
|
||||
run_mode: GraphRunMode,
|
||||
) -> ReferenceGraphRuntime:
|
||||
"""Instantiate every admitted provider from explicit immutable inputs."""
|
||||
|
||||
config = _load_graph_config(paths.graph_config)
|
||||
pinned_files = {
|
||||
ProviderRole.SOURCE: paths.baseline_profile,
|
||||
ProviderRole.DETECTOR: paths.baseline_profile,
|
||||
ProviderRole.GEOMETRY: paths.geometry_profile,
|
||||
ProviderRole.TEMPORAL: paths.temporal_motion_profile,
|
||||
ProviderRole.MOTION: paths.temporal_motion_profile,
|
||||
ProviderRole.ROLLING: paths.rolling_map_profile,
|
||||
ProviderRole.THREAT: paths.threat_profile,
|
||||
}
|
||||
_validate_provider_digests(config, pinned_files)
|
||||
|
||||
load_m4_baseline(paths.baseline_profile)
|
||||
geometry_profile = load_geometry_profile(paths.geometry_profile)
|
||||
temporal_motion_profile = load_temporal_motion_profile(paths.temporal_motion_profile)
|
||||
rolling_map_profile = load_rolling_map_profile(paths.rolling_map_profile)
|
||||
threat_profile = load_replay_threat_profile(paths.threat_profile)
|
||||
|
||||
source = DecodedRecordedSource(
|
||||
source=RecordedRavnoves00Source(
|
||||
camera_index_path=paths.camera_index,
|
||||
source_pack_path=paths.source_pack,
|
||||
pacing=(
|
||||
ReplayPacing.ONE_X
|
||||
if run_mode is GraphRunMode.SOURCE_PACED_LATEST_WINS
|
||||
else ReplayPacing.UNCAPPED
|
||||
),
|
||||
),
|
||||
decoder=PyAvRecordedImageDecoder(paths.video),
|
||||
)
|
||||
backend = TritonHttpInferenceBackend(triton_origin)
|
||||
try:
|
||||
store = RecordedGeometryStore(
|
||||
source_pack_path=paths.source_pack,
|
||||
local_surface_path=paths.local_surface,
|
||||
profile=geometry_profile,
|
||||
)
|
||||
body_frame_resolver = RecordedReplayBodyFrameResolver(
|
||||
store,
|
||||
profile=threat_profile.body_frame,
|
||||
)
|
||||
graph = ReferencePerceptionGraphV2(
|
||||
config=config,
|
||||
source=source,
|
||||
detector=FrozenYoloxDetectorProvider(
|
||||
mask=load_valid_fov_mask(paths.valid_fov_mask),
|
||||
backend=backend,
|
||||
),
|
||||
geometry=Ravnoves00GeometryAssociationProvider(store=store),
|
||||
temporal=BoundedSpatialTemporalProvider(
|
||||
point_resolver=store,
|
||||
profile=temporal_motion_profile,
|
||||
),
|
||||
motion=ClassIndependentMotionEstimator(profile=temporal_motion_profile),
|
||||
rolling=RollingLocalObstacleMapProvider(
|
||||
pose_resolver=store,
|
||||
profile=rolling_map_profile,
|
||||
),
|
||||
threat=DualEvidenceReplayThreatProvider(
|
||||
body_frame_resolver=body_frame_resolver,
|
||||
profile=threat_profile,
|
||||
),
|
||||
run_mode=run_mode,
|
||||
)
|
||||
except Exception:
|
||||
backend.close()
|
||||
raise
|
||||
return ReferenceGraphRuntime(graph=graph, inference_backend=backend)
|
||||
|
||||
|
||||
def _load_graph_config(path: Path) -> ReferencePerceptionGraphConfigV2:
|
||||
try:
|
||||
document = json.loads(path.resolve(strict=True).read_text("utf-8"))
|
||||
return ReferencePerceptionGraphConfigV2.from_dict(document)
|
||||
except (OSError, json.JSONDecodeError, ValueError) as exc:
|
||||
raise ReferenceGraphRuntimeError("reference graph config is invalid") from exc
|
||||
|
||||
|
||||
def _validate_provider_digests(
|
||||
config: ReferencePerceptionGraphConfigV2,
|
||||
pinned_files: dict[ProviderRole, Path],
|
||||
) -> None:
|
||||
pins = {pin.role: pin for pin in config.providers}
|
||||
if set(pins) != set(pinned_files):
|
||||
raise ReferenceGraphRuntimeError("reference graph provider pins are incomplete")
|
||||
for role, path in pinned_files.items():
|
||||
if _sha256_file(path) != pins[role].sha256:
|
||||
raise ReferenceGraphRuntimeError(f"{role.value} provider profile digest changed")
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
try:
|
||||
resolved = path.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise ReferenceGraphRuntimeError("pinned graph input is missing") from exc
|
||||
if resolved.is_symlink() or not resolved.is_file():
|
||||
raise ReferenceGraphRuntimeError("pinned graph input must be a regular file")
|
||||
digest = hashlib.sha256()
|
||||
with resolved.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ReferenceGraphRuntime",
|
||||
"ReferenceGraphRuntimeError",
|
||||
"ReferenceGraphRuntimePaths",
|
||||
"build_reference_graph_runtime",
|
||||
]
|
||||
Reference in New Issue
Block a user