SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(laboratory): compare M48S runtime hardening

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-25 19:38:11 +03:00
parent 84624ae3ea
commit 078b8421a6
8 changed files with 445 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/laboratory/m48s_fixed_class_detector_lab.py
+201 -16
View File
@@ -21,6 +21,7 @@ LAB_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-lab/v1"
CATALOG_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame-catalog/v1"
FRAME_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame/v1"
REPORT_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-report/v1"
RUNTIME_HARDENING_SCHEMA: Final = "missioncore.m48s-runtime-hardening-comparison/v1"
METHOD_SCHEMA: Final = "missioncore.laboratory-method/v1"
RESULT_PREFIX: Final = "m48s-fixed-class-detector-lab-"
TOURNAMENT_ID: Final = (
@@ -35,10 +36,35 @@ REFERENCE_GRAPH_ID: Final = (
"e8da7a521768daba0ead1a6e4803871ce3a85f91a7d8ee36c5719ac10433e791"
)
REFERENCE_GRAPH_REPLAY_ID: Final = (
"m48s-reference-graph-replay-"
"16d69d610c22e6f42071b8378cd75dfa6b95db4ceb800f9c7508fa3673504478"
"m48s-reference-graph-replay-16d69d610c22e6f42071b8378cd75dfa6b95db4ceb800f9c7508fa3673504478"
)
INTEGRATED_STATUS: Final = "complete-reference-graph-shadow-passed-production-not-authorized"
RUNTIME_HARDENING_RUNS: Final = {
"baseline": {
"directory": "m48s-pipeline-timing-c51761b5",
"schema_version": "missioncore.m48s-reference-graph-shadow-load/v1",
"result_sha256": "277f01a1b6499f08f6fdb65485296c9a6ca214a4f89deaca51d1dc9b00a28528",
"frames_sha256": "c870814159f362e3080ad72a44aab31e1210d5b2d2fd5da8390c05dcd30fbcb8",
"admitted_count": 4489,
"frame_count": 4480,
},
"hardened": {
"directory": "m48s-gc-bounded-477886d",
"schema_version": "missioncore.m48s-reference-graph-shadow-load/v2",
"result_sha256": "13e4423a89b0c1cc78b0219bf5842c42503951ef1372a6996818b3ed8ad3d3ed",
"frames_sha256": "1508da7570fc27aeed010a04ff5eadb6e4a8dedd31e9c617bc021c505a88513d",
"admitted_count": 4489,
"frame_count": 4488,
},
"prewarmed": {
"directory": "m48s-prewarm-84624ae",
"schema_version": "missioncore.m48s-reference-graph-shadow-load/v3",
"result_sha256": "5881f86ac01dc4e8886c4daef5bcd9d3510d50d7f4144c0341e0144ebb7015b1",
"frames_sha256": "67c041beee642f3ff8591c75c80a9bb56c29ad3d63168b38df19ecc475c99d55",
"admitted_count": 1000,
"frame_count": 1000,
},
}
YOLOX_ID: Final = (
"m48s-yolox-all-coco-shadow-7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06"
)
@@ -115,6 +141,14 @@ def build_m48s_fixed_class_detector_lab(
reference_graph_replay_path = reference_graph_replay_root / "manifest.json"
reference_graph_replay_worker_path = reference_graph_replay_root / "worker-result.json"
reference_graph_replay_frames_path = reference_graph_replay_root / "frames.jsonl"
hardening_root = runtime / "reference-graph-replay-results"
hardening_paths = {
name: {
"result": hardening_root / str(definition["directory"]) / "result.json",
"frames": hardening_root / str(definition["directory"]) / "frames.jsonl",
}
for name, definition in RUNTIME_HARDENING_RUNS.items()
}
yolox_root = runtime / "yolox-all-coco-results" / YOLOX_ID
yolox_manifest_path = yolox_root / "manifest.json"
yolox_frames_path = yolox_root / "frames.jsonl"
@@ -137,6 +171,7 @@ def build_m48s_fixed_class_detector_lab(
dfine_path,
rf_detr_path,
profile_path,
*(path for paths in hardening_paths.values() for path in paths.values()),
):
if path.is_symlink() or not path.is_file():
raise M48SFixedClassDetectorLabError(
@@ -150,6 +185,14 @@ def build_m48s_fixed_class_detector_lab(
reference_graph_worker = _read_object(reference_graph_worker_path)
reference_graph_replay = _read_object(reference_graph_replay_path)
reference_graph_replay_worker = _read_object(reference_graph_replay_worker_path)
hardening_runs = {
name: _read_object(paths["result"]) for name, paths in hardening_paths.items()
}
hardening_first_frames = {
name: _read_first_jsonl_object(paths["frames"])
for name, paths in hardening_paths.items()
if name in {"hardened", "prewarmed"}
}
yolox_manifest = _read_object(yolox_manifest_path)
dfine = _read_object(dfine_path)
rf_detr = _read_object(rf_detr_path)
@@ -169,6 +212,9 @@ def build_m48s_fixed_class_detector_lab(
rf_detr=rf_detr,
profile=profile,
yolox_frames=yolox_frames,
hardening_runs=hardening_runs,
hardening_first_frames=hardening_first_frames,
hardening_paths=hardening_paths,
)
source_paths = {frame_id: source_root / f"frame-{frame_id}.jpg" for frame_id in FRAME_IDS}
@@ -221,15 +267,16 @@ def build_m48s_fixed_class_detector_lab(
"reference_graph_document_sha256": sha256_path(reference_graph_path),
"reference_graph_worker_sha256": sha256_path(reference_graph_worker_path),
"reference_graph_replay_result_id": REFERENCE_GRAPH_REPLAY_ID,
"reference_graph_replay_document_sha256": sha256_path(
reference_graph_replay_path
),
"reference_graph_replay_worker_sha256": sha256_path(
reference_graph_replay_worker_path
),
"reference_graph_replay_frames_sha256": sha256_path(
reference_graph_replay_frames_path
),
"reference_graph_replay_document_sha256": sha256_path(reference_graph_replay_path),
"reference_graph_replay_worker_sha256": sha256_path(reference_graph_replay_worker_path),
"reference_graph_replay_frames_sha256": sha256_path(reference_graph_replay_frames_path),
"runtime_hardening": {
name: {
"result_sha256": sha256_path(paths["result"]),
"frames_sha256": sha256_path(paths["frames"]),
}
for name, paths in hardening_paths.items()
},
"yolox_result_id": YOLOX_ID,
"yolox_document_sha256": sha256_path(yolox_manifest_path),
},
@@ -238,7 +285,7 @@ def build_m48s_fixed_class_detector_lab(
}
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
result_id = RESULT_PREFIX + identity_sha256
completed_utc_ns = reference_graph_replay_worker.get("completed_utc_ns")
completed_utc_ns = hardening_runs["prewarmed"].get("completed_utc_ns")
if not isinstance(completed_utc_ns, int) or isinstance(completed_utc_ns, bool):
raise M48SFixedClassDetectorLabError(
"complete reference-graph completion time is unavailable"
@@ -260,6 +307,8 @@ def build_m48s_fixed_class_detector_lab(
load=load,
reference_graph=reference_graph,
candidates=candidates,
hardening_runs=hardening_runs,
hardening_first_frames=hardening_first_frames,
)
decision = {
"bounded_question_accepted": True,
@@ -283,8 +332,8 @@ def build_m48s_fixed_class_detector_lab(
"authority."
),
(
"Eight source frames were superseded by the qualified latest-wins graph; their "
"camera/LiDAR source evidence remains visible without invented world state."
"The visual timeline is the original qualified semantic replay; runtime-hardening "
"metrics come from separately sealed, input-identical replay runs."
),
]
@@ -370,6 +419,18 @@ def build_m48s_fixed_class_detector_lab(
reference_graph_replay_frames_path,
temporary / "reference-graph-replay-frames.jsonl",
)
for name, paths in hardening_paths.items():
shutil.copyfile(paths["result"], temporary / f"runtime-hardening-{name}.json")
startup_evidence = {
"schema_version": RUNTIME_HARDENING_SCHEMA,
"source_frames_sha256": {
name: sha256_path(hardening_paths[name]["frames"])
for name in ("hardened", "prewarmed")
},
"first_delivered_frames": hardening_first_frames,
}
startup_path = temporary / "runtime-hardening-startup.json"
startup_path.write_bytes(canonical_json(startup_evidence) + b"\n")
report = {
"schema_version": REPORT_SCHEMA,
"result_id": result_id,
@@ -379,6 +440,7 @@ def build_m48s_fixed_class_detector_lab(
"execution": {
"detector_load": load["execution"],
"complete_reference_graph": reference_graph["identity"]["evidence"]["execution"],
"runtime_hardening": metrics["runtime_hardening"],
},
"metrics": metrics,
"acceptance": {
@@ -450,6 +512,9 @@ def _validate_inputs(
rf_detr: dict[str, Any],
profile: dict[str, Any],
yolox_frames: list[dict[str, Any]],
hardening_runs: dict[str, dict[str, Any]],
hardening_first_frames: dict[str, dict[str, Any]],
hardening_paths: dict[str, dict[str, Path]],
) -> None:
decision = deployment.get("decision")
graph_identity = reference_graph.get("identity")
@@ -460,8 +525,7 @@ def _validate_inputs(
replay_artifacts = reference_graph_replay.get("artifacts")
replay_frame_summary = (
replay_identity.get("evidence", {}).get("frame_summary")
if isinstance(replay_identity, dict)
and isinstance(replay_identity.get("evidence"), dict)
if isinstance(replay_identity, dict) and isinstance(replay_identity.get("evidence"), dict)
else None
)
if (
@@ -539,6 +603,57 @@ def _validate_inputs(
authority = document.get("authority")
if authority is not None and authority != false_authority():
raise M48SFixedClassDetectorLabError("sealed M4.8S evidence gained authority")
for name, definition in RUNTIME_HARDENING_RUNS.items():
run = hardening_runs.get(name)
paths = hardening_paths.get(name)
if not isinstance(run, dict) or not isinstance(paths, dict):
raise M48SFixedClassDetectorLabError("runtime-hardening evidence is incomplete")
checks = run.get("checks")
execution = run.get("execution")
identity = run.get("identity")
frame_evidence = execution.get("frame_evidence") if isinstance(execution, dict) else None
if (
run.get("schema_version") != definition["schema_version"]
or run.get("completed") is not True
or run.get("integrated_runtime_gate_passed") is not True
or run.get("production_accepted") is not False
or run.get("authority") != false_authority()
or not isinstance(checks, dict)
or not checks
or not all(value is True for value in checks.values())
or not isinstance(identity, dict)
or identity.get("graph_id") != "reference-perception-graph/v2"
or identity.get("detector_provider_id")
!= "triton-rf-detr-large-coco-risk-fp16-shadow/v0"
or identity.get("worker_id") != "worker-006"
or not isinstance(execution, dict)
or execution.get("admitted_frames") != definition["admitted_count"]
or not isinstance(frame_evidence, dict)
or frame_evidence.get("row_count") != definition["frame_count"]
or frame_evidence.get("sha256") != definition["frames_sha256"]
or sha256_path(paths["result"]) != definition["result_sha256"]
or sha256_path(paths["frames"]) != definition["frames_sha256"]
):
raise M48SFixedClassDetectorLabError(
f"sealed runtime-hardening evidence changed: {name}"
)
for name in ("hardened", "prewarmed"):
frame = hardening_first_frames.get(name)
timing = frame.get("pipeline_timing") if isinstance(frame, dict) else None
detector = timing.get("detector") if isinstance(timing, dict) else None
if (
not isinstance(frame, dict)
or frame.get("schema_version") != "missioncore.m48s-reference-graph-frame-evidence/v1"
or not isinstance(timing, dict)
or timing.get("sequence") != 0
or not isinstance(timing.get("graph_admission_to_delivery_ns"), int)
or not isinstance(detector, dict)
or detector.get("sequence") != 0
or not isinstance(detector.get("total_duration_ns"), int)
):
raise M48SFixedClassDetectorLabError(
f"runtime-hardening startup evidence changed: {name}"
)
def _method(
@@ -674,6 +789,8 @@ def _metrics(
load: dict[str, Any],
reference_graph: dict[str, Any],
candidates: list[dict[str, object]],
hardening_runs: dict[str, dict[str, Any]],
hardening_first_frames: dict[str, dict[str, Any]],
) -> dict[str, object]:
graph_evidence = reference_graph["identity"]["evidence"]
graph_execution = graph_evidence["execution"]
@@ -739,6 +856,57 @@ def _metrics(
for key in ("failed", "stale", "rejected", "unavailable")
),
},
"runtime_hardening": _runtime_hardening_metrics(
runs=hardening_runs,
first_frames=hardening_first_frames,
),
}
def _runtime_hardening_metrics(
*,
runs: dict[str, dict[str, Any]],
first_frames: dict[str, dict[str, Any]],
) -> dict[str, object]:
def full_run(name: str) -> dict[str, object]:
run = runs[name]
execution = run["execution"]
completion = run["metrics"]["world_state_completion_age_ms"]
pipeline = run["metrics"]["pipeline_timing"]
terminal = execution["terminal_outcomes"]
return {
"source_frames_admitted": execution["admitted_frames"],
"delivered_world_states": execution["delivered_world_states"],
"superseded_frames": terminal.get("superseded", 0),
"effective_world_state_fps": execution["effective_world_state_fps"],
"world_state_completion_age_p95_ms": completion["p95"],
"world_state_completion_age_p99_ms": completion["p99"],
"world_state_completion_age_maximum_ms": completion["maximum"],
"rolling_maximum_ms": pipeline["provider_ms"]["rolling"]["maximum"],
"geometry_maximum_ms": pipeline["provider_ms"]["geometry"]["maximum"],
"additional_inference_passes": pipeline["additional_inference_passes"],
}
def startup_frame(name: str) -> dict[str, float]:
timing = first_frames[name]["pipeline_timing"]
return {
"detector_ms": timing["detector"]["total_duration_ns"] / 1_000_000,
"world_state_ms": timing["graph_admission_to_delivery_ns"] / 1_000_000,
}
prewarmed_loop = runs["prewarmed"]["execution"]["loops"][0]
warmup = prewarmed_loop["detector_warmup"]
return {
"schema_version": RUNTIME_HARDENING_SCHEMA,
"baseline": full_run("baseline"),
"hardened": full_run("hardened"),
"startup": {
"baseline": startup_frame("hardened"),
"prewarmed": startup_frame("prewarmed"),
"prewarm_duration_ms": warmup["total_duration_ns"] / 1_000_000,
"prewarm_inference_passes": warmup["inference_passes"],
"validation_frames": runs["prewarmed"]["execution"]["admitted_frames"],
},
}
@@ -843,6 +1011,11 @@ def _artifact_manifest(root: Path) -> list[dict[str, object]]:
media_type = "application/x-ndjson"
schema_version = "missioncore.m48s-reference-graph-frame-evidence/v0"
role = "visual-evidence-full-replay-world-state"
elif relative.startswith("runtime-hardening-"):
schema_version = (
RUNTIME_HARDENING_SCHEMA if relative == "runtime-hardening-startup.json" else None
)
role = "upstream-runtime-hardening-evidence"
artifacts.append(
{
"role": role,
@@ -866,6 +1039,18 @@ def _read_object(path: Path) -> dict[str, Any]:
return value
def _read_first_jsonl_object(path: Path) -> dict[str, Any]:
try:
with path.open("r", encoding="utf-8") as stream:
line = stream.readline()
value = json.loads(line)
except (OSError, json.JSONDecodeError) as exc:
raise M48SFixedClassDetectorLabError(f"invalid first-frame evidence: {path.name}") from exc
if not isinstance(value, dict):
raise M48SFixedClassDetectorLabError(f"first-frame evidence must be an object: {path.name}")
return value
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
try:
rows = [json.loads(line) for line in path.read_text("utf-8").splitlines() if line]
src/k1link/web/m48s_fixed_class_detector_lab_api.py
+61
View File
@@ -337,6 +337,10 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
or method.get("completeness") != "complete"
or not isinstance(metrics, dict)
or (integrated and not isinstance(metrics.get("integrated_world_state"), dict))
or (
metrics.get("runtime_hardening") is not None
and not _valid_runtime_hardening(metrics["runtime_hardening"])
)
or not isinstance(manifest.get("limitations"), list)
or not isinstance(catalog_descriptor, dict)
or catalog_descriptor.get("path") != "catalog.json"
@@ -363,6 +367,63 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
return {"manifest": manifest, "catalog": catalog}
def _valid_runtime_hardening(value: object) -> bool:
if not isinstance(value, dict) or value.get("schema_version") != (
"missioncore.m48s-runtime-hardening-comparison/v1"
):
return False
full_run_keys = {
"source_frames_admitted",
"delivered_world_states",
"superseded_frames",
"effective_world_state_fps",
"world_state_completion_age_p95_ms",
"world_state_completion_age_p99_ms",
"world_state_completion_age_maximum_ms",
"rolling_maximum_ms",
"geometry_maximum_ms",
"additional_inference_passes",
}
for name in ("baseline", "hardened"):
run = value.get(name)
if (
not isinstance(run, dict)
or set(run) != full_run_keys
or any(not _nonnegative_number(item) for item in run.values())
):
return False
startup = value.get("startup")
if not isinstance(startup, dict) or set(startup) != {
"baseline",
"prewarmed",
"prewarm_duration_ms",
"prewarm_inference_passes",
"validation_frames",
}:
return False
for name in ("baseline", "prewarmed"):
frame = startup.get(name)
if (
not isinstance(frame, dict)
or set(frame) != {"detector_ms", "world_state_ms"}
or any(not _nonnegative_number(item) for item in frame.values())
):
return False
return (
_nonnegative_number(startup.get("prewarm_duration_ms"))
and isinstance(startup.get("prewarm_inference_passes"), int)
and not isinstance(startup.get("prewarm_inference_passes"), bool)
and startup["prewarm_inference_passes"] > 0
and isinstance(startup.get("validation_frames"), int)
and not isinstance(startup.get("validation_frames"), bool)
and startup["validation_frames"] > 0
)
def _nonnegative_number(value: object) -> bool:
return isinstance(value, (int, float)) and not isinstance(value, bool) and float(value) >= 0.0
def _candidate_signature(candidate: Path) -> tuple[int, ...]:
if not candidate.is_dir() or candidate.is_symlink():
raise RuntimeError("M4.8S result candidate is invalid")