feat(laboratory): compare M48S runtime hardening
This commit is contained in:
@@ -337,6 +337,10 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
|
||||
or method.get("completeness") != "complete"
|
||||
or not isinstance(metrics, dict)
|
||||
or (integrated and not isinstance(metrics.get("integrated_world_state"), dict))
|
||||
or (
|
||||
metrics.get("runtime_hardening") is not None
|
||||
and not _valid_runtime_hardening(metrics["runtime_hardening"])
|
||||
)
|
||||
or not isinstance(manifest.get("limitations"), list)
|
||||
or not isinstance(catalog_descriptor, dict)
|
||||
or catalog_descriptor.get("path") != "catalog.json"
|
||||
@@ -363,6 +367,63 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
|
||||
return {"manifest": manifest, "catalog": catalog}
|
||||
|
||||
|
||||
def _valid_runtime_hardening(value: object) -> bool:
|
||||
if not isinstance(value, dict) or value.get("schema_version") != (
|
||||
"missioncore.m48s-runtime-hardening-comparison/v1"
|
||||
):
|
||||
return False
|
||||
full_run_keys = {
|
||||
"source_frames_admitted",
|
||||
"delivered_world_states",
|
||||
"superseded_frames",
|
||||
"effective_world_state_fps",
|
||||
"world_state_completion_age_p95_ms",
|
||||
"world_state_completion_age_p99_ms",
|
||||
"world_state_completion_age_maximum_ms",
|
||||
"rolling_maximum_ms",
|
||||
"geometry_maximum_ms",
|
||||
"additional_inference_passes",
|
||||
}
|
||||
for name in ("baseline", "hardened"):
|
||||
run = value.get(name)
|
||||
if (
|
||||
not isinstance(run, dict)
|
||||
or set(run) != full_run_keys
|
||||
or any(not _nonnegative_number(item) for item in run.values())
|
||||
):
|
||||
return False
|
||||
startup = value.get("startup")
|
||||
if not isinstance(startup, dict) or set(startup) != {
|
||||
"baseline",
|
||||
"prewarmed",
|
||||
"prewarm_duration_ms",
|
||||
"prewarm_inference_passes",
|
||||
"validation_frames",
|
||||
}:
|
||||
return False
|
||||
for name in ("baseline", "prewarmed"):
|
||||
frame = startup.get(name)
|
||||
if (
|
||||
not isinstance(frame, dict)
|
||||
or set(frame) != {"detector_ms", "world_state_ms"}
|
||||
or any(not _nonnegative_number(item) for item in frame.values())
|
||||
):
|
||||
return False
|
||||
return (
|
||||
_nonnegative_number(startup.get("prewarm_duration_ms"))
|
||||
and isinstance(startup.get("prewarm_inference_passes"), int)
|
||||
and not isinstance(startup.get("prewarm_inference_passes"), bool)
|
||||
and startup["prewarm_inference_passes"] > 0
|
||||
and isinstance(startup.get("validation_frames"), int)
|
||||
and not isinstance(startup.get("validation_frames"), bool)
|
||||
and startup["validation_frames"] > 0
|
||||
)
|
||||
|
||||
|
||||
def _nonnegative_number(value: object) -> bool:
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool) and float(value) >= 0.0
|
||||
|
||||
|
||||
def _candidate_signature(candidate: Path) -> tuple[int, ...]:
|
||||
if not candidate.is_dir() or candidate.is_symlink():
|
||||
raise RuntimeError("M4.8S result candidate is invalid")
|
||||
|
||||
Reference in New Issue
Block a user