diff --git a/apps/control-station/src/core/laboratory/m48sFixedClassDetector.ts b/apps/control-station/src/core/laboratory/m48sFixedClassDetector.ts index 454b216..f1974ee 100644 --- a/apps/control-station/src/core/laboratory/m48sFixedClassDetector.ts +++ b/apps/control-station/src/core/laboratory/m48sFixedClassDetector.ts @@ -80,6 +80,31 @@ export interface M48SIntegratedWorldState { failures: number; } +export interface M48SRuntimeHardeningFullRun { + sourceFramesAdmitted: number; + deliveredWorldStates: number; + supersededFrames: number; + effectiveWorldStateFps: number; + worldStateCompletionAgeP95Ms: number; + worldStateCompletionAgeP99Ms: number; + worldStateCompletionAgeMaximumMs: number; + rollingMaximumMs: number; + geometryMaximumMs: number; + additionalInferencePasses: number; +} + +export interface M48SRuntimeHardeningComparison { + baseline: M48SRuntimeHardeningFullRun; + hardened: M48SRuntimeHardeningFullRun; + startup: { + baseline: { detectorMs: number; worldStateMs: number }; + prewarmed: { detectorMs: number; worldStateMs: number }; + prewarmDurationMs: number; + prewarmInferencePasses: number; + validationFrames: number; + }; +} + export interface M48SFixedClassDetectorResult { resultId: string; createdAtUtc: string; @@ -118,6 +143,7 @@ export interface M48SFixedClassDetectorResult { failures: number; }; integratedWorldState: M48SIntegratedWorldState | null; + runtimeHardening: M48SRuntimeHardeningComparison | null; }; decision: { selectedCandidate: "rf-detr"; @@ -366,6 +392,71 @@ function integratedWorldStateValue(value: unknown): M48SIntegratedWorldState { }; } +function runtimeHardeningFullRunValue( + value: unknown, + label: string, +): M48SRuntimeHardeningFullRun { + const run = objectValue(value, label); + return { + sourceFramesAdmitted: integerValue(run.source_frames_admitted, `${label}.source_frames_admitted`), + deliveredWorldStates: integerValue(run.delivered_world_states, `${label}.delivered_world_states`), + supersededFrames: integerValue(run.superseded_frames, `${label}.superseded_frames`), + effectiveWorldStateFps: numberValue(run.effective_world_state_fps, `${label}.effective_world_state_fps`), + worldStateCompletionAgeP95Ms: numberValue(run.world_state_completion_age_p95_ms, `${label}.world_state_completion_age_p95_ms`), + worldStateCompletionAgeP99Ms: numberValue(run.world_state_completion_age_p99_ms, `${label}.world_state_completion_age_p99_ms`), + worldStateCompletionAgeMaximumMs: numberValue(run.world_state_completion_age_maximum_ms, `${label}.world_state_completion_age_maximum_ms`), + rollingMaximumMs: numberValue(run.rolling_maximum_ms, `${label}.rolling_maximum_ms`), + geometryMaximumMs: numberValue(run.geometry_maximum_ms, `${label}.geometry_maximum_ms`), + additionalInferencePasses: integerValue(run.additional_inference_passes, `${label}.additional_inference_passes`), + }; +} + +function runtimeHardeningComparisonValue(value: unknown): M48SRuntimeHardeningComparison { + const comparison = objectValue(value, "M4.8S.metrics.runtime_hardening"); + exact( + comparison.schema_version, + "missioncore.m48s-runtime-hardening-comparison/v1", + "M4.8S.metrics.runtime_hardening.schema_version", + ); + const startup = objectValue(comparison.startup, "M4.8S.runtime_hardening.startup"); + const startupFrame = ( + candidate: unknown, + label: string, + ): { detectorMs: number; worldStateMs: number } => { + const frame = objectValue(candidate, label); + return { + detectorMs: numberValue(frame.detector_ms, `${label}.detector_ms`), + worldStateMs: numberValue(frame.world_state_ms, `${label}.world_state_ms`), + }; + }; + return { + baseline: runtimeHardeningFullRunValue( + comparison.baseline, + "M4.8S.runtime_hardening.baseline", + ), + hardened: runtimeHardeningFullRunValue( + comparison.hardened, + "M4.8S.runtime_hardening.hardened", + ), + startup: { + baseline: startupFrame(startup.baseline, "M4.8S.runtime_hardening.startup.baseline"), + prewarmed: startupFrame(startup.prewarmed, "M4.8S.runtime_hardening.startup.prewarmed"), + prewarmDurationMs: numberValue( + startup.prewarm_duration_ms, + "M4.8S.runtime_hardening.startup.prewarm_duration_ms", + ), + prewarmInferencePasses: integerValue( + startup.prewarm_inference_passes, + "M4.8S.runtime_hardening.startup.prewarm_inference_passes", + ), + validationFrames: integerValue( + startup.validation_frames, + "M4.8S.runtime_hardening.startup.validation_frames", + ), + }, + }; +} + function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorResult { const payload = objectValue(value, "M4.8S"); exact( @@ -457,6 +548,9 @@ function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorRe integratedWorldState: integratedGate ? integratedWorldStateValue(metrics.integrated_world_state) : null, + runtimeHardening: metrics.runtime_hardening === undefined + ? null + : runtimeHardeningComparisonValue(metrics.runtime_hardening), }, decision: { selectedCandidate: "rf-detr", diff --git a/apps/control-station/src/workspaces/laboratory/M48SFixedClassDetectorResult.tsx b/apps/control-station/src/workspaces/laboratory/M48SFixedClassDetectorResult.tsx index 3618d78..c7b93d9 100644 --- a/apps/control-station/src/workspaces/laboratory/M48SFixedClassDetectorResult.tsx +++ b/apps/control-station/src/workspaces/laboratory/M48SFixedClassDetectorResult.tsx @@ -21,7 +21,10 @@ export function M48SFixedClassDetectorResultView({ const selected = result.metrics.candidates.find((candidate) => candidate.selected); const load = result.metrics.detectorLoad; const integrated = result.metrics.integratedWorldState; - const status = integrated + const hardening = result.metrics.runtimeHardening; + const status = hardening + ? `После hardening: ${hardening.hardened.deliveredWorldStates.toLocaleString("ru-RU")} из ${hardening.hardened.sourceFramesAdmitted.toLocaleString("ru-RU")} world states доставлены` + : integrated ? "Полный RF-DETR reference graph выдержал realtime shadow" : "RF-DETR-L выдержал detector-only realtime shadow"; return ( @@ -29,7 +32,7 @@ export function M48SFixedClassDetectorResultView({ summary={( )} diff --git a/apps/control-station/test/m48sFixedClassDetector.test.mjs b/apps/control-station/test/m48sFixedClassDetector.test.mjs index 712f31c..8730793 100644 --- a/apps/control-station/test/m48sFixedClassDetector.test.mjs +++ b/apps/control-station/test/m48sFixedClassDetector.test.mjs @@ -138,6 +138,40 @@ function resultPayload() { additional_inference_passes: 0, failures: 0, }, + runtime_hardening: { + schema_version: "missioncore.m48s-runtime-hardening-comparison/v1", + baseline: { + source_frames_admitted: 4489, + delivered_world_states: 4480, + superseded_frames: 9, + effective_world_state_fps: 9.750635, + world_state_completion_age_p95_ms: 76.886564, + world_state_completion_age_p99_ms: 100.661219, + world_state_completion_age_maximum_ms: 794.748644, + rolling_maximum_ms: 762.638263, + geometry_maximum_ms: 350.066302, + additional_inference_passes: 0, + }, + hardened: { + source_frames_admitted: 4489, + delivered_world_states: 4488, + superseded_frames: 1, + effective_world_state_fps: 9.821942, + world_state_completion_age_p95_ms: 75.270655, + world_state_completion_age_p99_ms: 94.309605, + world_state_completion_age_maximum_ms: 449.893287, + rolling_maximum_ms: 28.044958, + geometry_maximum_ms: 49.174149, + additional_inference_passes: 0, + }, + startup: { + baseline: { detector_ms: 397.356888, world_state_ms: 449.893287 }, + prewarmed: { detector_ms: 27.662887, world_state_ms: 60.516957 }, + prewarm_duration_ms: 420.721918, + prewarm_inference_passes: 1, + validation_frames: 1000, + }, + }, }, decision: { selected_candidate: "rf-detr", @@ -175,6 +209,9 @@ test("M4.8S result exposes complete graph load without production authority", as assert.equal(result.metrics.integratedWorldState.deliveredWorldStates, 4481); assert.equal(result.metrics.integratedWorldState.worldStateCompletionAgeP95Ms, 74.733648); assert.equal(result.metrics.integratedWorldState.additionalInferencePasses, 0); + assert.equal(result.metrics.runtimeHardening.baseline.supersededFrames, 9); + assert.equal(result.metrics.runtimeHardening.hardened.supersededFrames, 1); + assert.equal(result.metrics.runtimeHardening.startup.prewarmed.worldStateMs, 60.516957); assert.equal(result.decision.integratedWorldStateGatePassed, true); assert.equal(result.decision.productionAccepted, false); assert.equal(result.authority.navigationOrSafetyAccepted, false); diff --git a/config/laboratory-value-review.json b/config/laboratory-value-review.json index 60d1664..4b8cdb7 100644 --- a/config/laboratory-value-review.json +++ b/config/laboratory-value-review.json @@ -249,7 +249,7 @@ }, { "catalog_id": "m48s-fixed-class-detector", - "evidence_id": "m48s-fixed-class-detector-lab-d1bac05a9e43d407b0f931105cc0e84183ef9ff37666911f03c41486beeb7ef9", + "evidence_id": "m48s-fixed-class-detector-lab-7411aadc35f5b61b97ee59c983e125e0a3781612d399c4dfa779b272eeb0e56e", "signal": "progress", "lifecycle": "current", "visual_evidence": "available" diff --git a/src/k1link/laboratory/m48s_fixed_class_detector_lab.py b/src/k1link/laboratory/m48s_fixed_class_detector_lab.py index d93c35d..d755845 100644 --- a/src/k1link/laboratory/m48s_fixed_class_detector_lab.py +++ b/src/k1link/laboratory/m48s_fixed_class_detector_lab.py @@ -21,6 +21,7 @@ LAB_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-lab/v1" CATALOG_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame-catalog/v1" FRAME_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame/v1" REPORT_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-report/v1" +RUNTIME_HARDENING_SCHEMA: Final = "missioncore.m48s-runtime-hardening-comparison/v1" METHOD_SCHEMA: Final = "missioncore.laboratory-method/v1" RESULT_PREFIX: Final = "m48s-fixed-class-detector-lab-" TOURNAMENT_ID: Final = ( @@ -35,10 +36,35 @@ REFERENCE_GRAPH_ID: Final = ( "e8da7a521768daba0ead1a6e4803871ce3a85f91a7d8ee36c5719ac10433e791" ) REFERENCE_GRAPH_REPLAY_ID: Final = ( - "m48s-reference-graph-replay-" - "16d69d610c22e6f42071b8378cd75dfa6b95db4ceb800f9c7508fa3673504478" + "m48s-reference-graph-replay-16d69d610c22e6f42071b8378cd75dfa6b95db4ceb800f9c7508fa3673504478" ) INTEGRATED_STATUS: Final = "complete-reference-graph-shadow-passed-production-not-authorized" +RUNTIME_HARDENING_RUNS: Final = { + "baseline": { + "directory": "m48s-pipeline-timing-c51761b5", + "schema_version": "missioncore.m48s-reference-graph-shadow-load/v1", + "result_sha256": "277f01a1b6499f08f6fdb65485296c9a6ca214a4f89deaca51d1dc9b00a28528", + "frames_sha256": "c870814159f362e3080ad72a44aab31e1210d5b2d2fd5da8390c05dcd30fbcb8", + "admitted_count": 4489, + "frame_count": 4480, + }, + "hardened": { + "directory": "m48s-gc-bounded-477886d", + "schema_version": "missioncore.m48s-reference-graph-shadow-load/v2", + "result_sha256": "13e4423a89b0c1cc78b0219bf5842c42503951ef1372a6996818b3ed8ad3d3ed", + "frames_sha256": "1508da7570fc27aeed010a04ff5eadb6e4a8dedd31e9c617bc021c505a88513d", + "admitted_count": 4489, + "frame_count": 4488, + }, + "prewarmed": { + "directory": "m48s-prewarm-84624ae", + "schema_version": "missioncore.m48s-reference-graph-shadow-load/v3", + "result_sha256": "5881f86ac01dc4e8886c4daef5bcd9d3510d50d7f4144c0341e0144ebb7015b1", + "frames_sha256": "67c041beee642f3ff8591c75c80a9bb56c29ad3d63168b38df19ecc475c99d55", + "admitted_count": 1000, + "frame_count": 1000, + }, +} YOLOX_ID: Final = ( "m48s-yolox-all-coco-shadow-7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06" ) @@ -115,6 +141,14 @@ def build_m48s_fixed_class_detector_lab( reference_graph_replay_path = reference_graph_replay_root / "manifest.json" reference_graph_replay_worker_path = reference_graph_replay_root / "worker-result.json" reference_graph_replay_frames_path = reference_graph_replay_root / "frames.jsonl" + hardening_root = runtime / "reference-graph-replay-results" + hardening_paths = { + name: { + "result": hardening_root / str(definition["directory"]) / "result.json", + "frames": hardening_root / str(definition["directory"]) / "frames.jsonl", + } + for name, definition in RUNTIME_HARDENING_RUNS.items() + } yolox_root = runtime / "yolox-all-coco-results" / YOLOX_ID yolox_manifest_path = yolox_root / "manifest.json" yolox_frames_path = yolox_root / "frames.jsonl" @@ -137,6 +171,7 @@ def build_m48s_fixed_class_detector_lab( dfine_path, rf_detr_path, profile_path, + *(path for paths in hardening_paths.values() for path in paths.values()), ): if path.is_symlink() or not path.is_file(): raise M48SFixedClassDetectorLabError( @@ -150,6 +185,14 @@ def build_m48s_fixed_class_detector_lab( reference_graph_worker = _read_object(reference_graph_worker_path) reference_graph_replay = _read_object(reference_graph_replay_path) reference_graph_replay_worker = _read_object(reference_graph_replay_worker_path) + hardening_runs = { + name: _read_object(paths["result"]) for name, paths in hardening_paths.items() + } + hardening_first_frames = { + name: _read_first_jsonl_object(paths["frames"]) + for name, paths in hardening_paths.items() + if name in {"hardened", "prewarmed"} + } yolox_manifest = _read_object(yolox_manifest_path) dfine = _read_object(dfine_path) rf_detr = _read_object(rf_detr_path) @@ -169,6 +212,9 @@ def build_m48s_fixed_class_detector_lab( rf_detr=rf_detr, profile=profile, yolox_frames=yolox_frames, + hardening_runs=hardening_runs, + hardening_first_frames=hardening_first_frames, + hardening_paths=hardening_paths, ) source_paths = {frame_id: source_root / f"frame-{frame_id}.jpg" for frame_id in FRAME_IDS} @@ -221,15 +267,16 @@ def build_m48s_fixed_class_detector_lab( "reference_graph_document_sha256": sha256_path(reference_graph_path), "reference_graph_worker_sha256": sha256_path(reference_graph_worker_path), "reference_graph_replay_result_id": REFERENCE_GRAPH_REPLAY_ID, - "reference_graph_replay_document_sha256": sha256_path( - reference_graph_replay_path - ), - "reference_graph_replay_worker_sha256": sha256_path( - reference_graph_replay_worker_path - ), - "reference_graph_replay_frames_sha256": sha256_path( - reference_graph_replay_frames_path - ), + "reference_graph_replay_document_sha256": sha256_path(reference_graph_replay_path), + "reference_graph_replay_worker_sha256": sha256_path(reference_graph_replay_worker_path), + "reference_graph_replay_frames_sha256": sha256_path(reference_graph_replay_frames_path), + "runtime_hardening": { + name: { + "result_sha256": sha256_path(paths["result"]), + "frames_sha256": sha256_path(paths["frames"]), + } + for name, paths in hardening_paths.items() + }, "yolox_result_id": YOLOX_ID, "yolox_document_sha256": sha256_path(yolox_manifest_path), }, @@ -238,7 +285,7 @@ def build_m48s_fixed_class_detector_lab( } identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest() result_id = RESULT_PREFIX + identity_sha256 - completed_utc_ns = reference_graph_replay_worker.get("completed_utc_ns") + completed_utc_ns = hardening_runs["prewarmed"].get("completed_utc_ns") if not isinstance(completed_utc_ns, int) or isinstance(completed_utc_ns, bool): raise M48SFixedClassDetectorLabError( "complete reference-graph completion time is unavailable" @@ -260,6 +307,8 @@ def build_m48s_fixed_class_detector_lab( load=load, reference_graph=reference_graph, candidates=candidates, + hardening_runs=hardening_runs, + hardening_first_frames=hardening_first_frames, ) decision = { "bounded_question_accepted": True, @@ -283,8 +332,8 @@ def build_m48s_fixed_class_detector_lab( "authority." ), ( - "Eight source frames were superseded by the qualified latest-wins graph; their " - "camera/LiDAR source evidence remains visible without invented world state." + "The visual timeline is the original qualified semantic replay; runtime-hardening " + "metrics come from separately sealed, input-identical replay runs." ), ] @@ -370,6 +419,18 @@ def build_m48s_fixed_class_detector_lab( reference_graph_replay_frames_path, temporary / "reference-graph-replay-frames.jsonl", ) + for name, paths in hardening_paths.items(): + shutil.copyfile(paths["result"], temporary / f"runtime-hardening-{name}.json") + startup_evidence = { + "schema_version": RUNTIME_HARDENING_SCHEMA, + "source_frames_sha256": { + name: sha256_path(hardening_paths[name]["frames"]) + for name in ("hardened", "prewarmed") + }, + "first_delivered_frames": hardening_first_frames, + } + startup_path = temporary / "runtime-hardening-startup.json" + startup_path.write_bytes(canonical_json(startup_evidence) + b"\n") report = { "schema_version": REPORT_SCHEMA, "result_id": result_id, @@ -379,6 +440,7 @@ def build_m48s_fixed_class_detector_lab( "execution": { "detector_load": load["execution"], "complete_reference_graph": reference_graph["identity"]["evidence"]["execution"], + "runtime_hardening": metrics["runtime_hardening"], }, "metrics": metrics, "acceptance": { @@ -450,6 +512,9 @@ def _validate_inputs( rf_detr: dict[str, Any], profile: dict[str, Any], yolox_frames: list[dict[str, Any]], + hardening_runs: dict[str, dict[str, Any]], + hardening_first_frames: dict[str, dict[str, Any]], + hardening_paths: dict[str, dict[str, Path]], ) -> None: decision = deployment.get("decision") graph_identity = reference_graph.get("identity") @@ -460,8 +525,7 @@ def _validate_inputs( replay_artifacts = reference_graph_replay.get("artifacts") replay_frame_summary = ( replay_identity.get("evidence", {}).get("frame_summary") - if isinstance(replay_identity, dict) - and isinstance(replay_identity.get("evidence"), dict) + if isinstance(replay_identity, dict) and isinstance(replay_identity.get("evidence"), dict) else None ) if ( @@ -539,6 +603,57 @@ def _validate_inputs( authority = document.get("authority") if authority is not None and authority != false_authority(): raise M48SFixedClassDetectorLabError("sealed M4.8S evidence gained authority") + for name, definition in RUNTIME_HARDENING_RUNS.items(): + run = hardening_runs.get(name) + paths = hardening_paths.get(name) + if not isinstance(run, dict) or not isinstance(paths, dict): + raise M48SFixedClassDetectorLabError("runtime-hardening evidence is incomplete") + checks = run.get("checks") + execution = run.get("execution") + identity = run.get("identity") + frame_evidence = execution.get("frame_evidence") if isinstance(execution, dict) else None + if ( + run.get("schema_version") != definition["schema_version"] + or run.get("completed") is not True + or run.get("integrated_runtime_gate_passed") is not True + or run.get("production_accepted") is not False + or run.get("authority") != false_authority() + or not isinstance(checks, dict) + or not checks + or not all(value is True for value in checks.values()) + or not isinstance(identity, dict) + or identity.get("graph_id") != "reference-perception-graph/v2" + or identity.get("detector_provider_id") + != "triton-rf-detr-large-coco-risk-fp16-shadow/v0" + or identity.get("worker_id") != "worker-006" + or not isinstance(execution, dict) + or execution.get("admitted_frames") != definition["admitted_count"] + or not isinstance(frame_evidence, dict) + or frame_evidence.get("row_count") != definition["frame_count"] + or frame_evidence.get("sha256") != definition["frames_sha256"] + or sha256_path(paths["result"]) != definition["result_sha256"] + or sha256_path(paths["frames"]) != definition["frames_sha256"] + ): + raise M48SFixedClassDetectorLabError( + f"sealed runtime-hardening evidence changed: {name}" + ) + for name in ("hardened", "prewarmed"): + frame = hardening_first_frames.get(name) + timing = frame.get("pipeline_timing") if isinstance(frame, dict) else None + detector = timing.get("detector") if isinstance(timing, dict) else None + if ( + not isinstance(frame, dict) + or frame.get("schema_version") != "missioncore.m48s-reference-graph-frame-evidence/v1" + or not isinstance(timing, dict) + or timing.get("sequence") != 0 + or not isinstance(timing.get("graph_admission_to_delivery_ns"), int) + or not isinstance(detector, dict) + or detector.get("sequence") != 0 + or not isinstance(detector.get("total_duration_ns"), int) + ): + raise M48SFixedClassDetectorLabError( + f"runtime-hardening startup evidence changed: {name}" + ) def _method( @@ -674,6 +789,8 @@ def _metrics( load: dict[str, Any], reference_graph: dict[str, Any], candidates: list[dict[str, object]], + hardening_runs: dict[str, dict[str, Any]], + hardening_first_frames: dict[str, dict[str, Any]], ) -> dict[str, object]: graph_evidence = reference_graph["identity"]["evidence"] graph_execution = graph_evidence["execution"] @@ -739,6 +856,57 @@ def _metrics( for key in ("failed", "stale", "rejected", "unavailable") ), }, + "runtime_hardening": _runtime_hardening_metrics( + runs=hardening_runs, + first_frames=hardening_first_frames, + ), + } + + +def _runtime_hardening_metrics( + *, + runs: dict[str, dict[str, Any]], + first_frames: dict[str, dict[str, Any]], +) -> dict[str, object]: + def full_run(name: str) -> dict[str, object]: + run = runs[name] + execution = run["execution"] + completion = run["metrics"]["world_state_completion_age_ms"] + pipeline = run["metrics"]["pipeline_timing"] + terminal = execution["terminal_outcomes"] + return { + "source_frames_admitted": execution["admitted_frames"], + "delivered_world_states": execution["delivered_world_states"], + "superseded_frames": terminal.get("superseded", 0), + "effective_world_state_fps": execution["effective_world_state_fps"], + "world_state_completion_age_p95_ms": completion["p95"], + "world_state_completion_age_p99_ms": completion["p99"], + "world_state_completion_age_maximum_ms": completion["maximum"], + "rolling_maximum_ms": pipeline["provider_ms"]["rolling"]["maximum"], + "geometry_maximum_ms": pipeline["provider_ms"]["geometry"]["maximum"], + "additional_inference_passes": pipeline["additional_inference_passes"], + } + + def startup_frame(name: str) -> dict[str, float]: + timing = first_frames[name]["pipeline_timing"] + return { + "detector_ms": timing["detector"]["total_duration_ns"] / 1_000_000, + "world_state_ms": timing["graph_admission_to_delivery_ns"] / 1_000_000, + } + + prewarmed_loop = runs["prewarmed"]["execution"]["loops"][0] + warmup = prewarmed_loop["detector_warmup"] + return { + "schema_version": RUNTIME_HARDENING_SCHEMA, + "baseline": full_run("baseline"), + "hardened": full_run("hardened"), + "startup": { + "baseline": startup_frame("hardened"), + "prewarmed": startup_frame("prewarmed"), + "prewarm_duration_ms": warmup["total_duration_ns"] / 1_000_000, + "prewarm_inference_passes": warmup["inference_passes"], + "validation_frames": runs["prewarmed"]["execution"]["admitted_frames"], + }, } @@ -843,6 +1011,11 @@ def _artifact_manifest(root: Path) -> list[dict[str, object]]: media_type = "application/x-ndjson" schema_version = "missioncore.m48s-reference-graph-frame-evidence/v0" role = "visual-evidence-full-replay-world-state" + elif relative.startswith("runtime-hardening-"): + schema_version = ( + RUNTIME_HARDENING_SCHEMA if relative == "runtime-hardening-startup.json" else None + ) + role = "upstream-runtime-hardening-evidence" artifacts.append( { "role": role, @@ -866,6 +1039,18 @@ def _read_object(path: Path) -> dict[str, Any]: return value +def _read_first_jsonl_object(path: Path) -> dict[str, Any]: + try: + with path.open("r", encoding="utf-8") as stream: + line = stream.readline() + value = json.loads(line) + except (OSError, json.JSONDecodeError) as exc: + raise M48SFixedClassDetectorLabError(f"invalid first-frame evidence: {path.name}") from exc + if not isinstance(value, dict): + raise M48SFixedClassDetectorLabError(f"first-frame evidence must be an object: {path.name}") + return value + + def _read_jsonl(path: Path) -> list[dict[str, Any]]: try: rows = [json.loads(line) for line in path.read_text("utf-8").splitlines() if line] diff --git a/src/k1link/web/m48s_fixed_class_detector_lab_api.py b/src/k1link/web/m48s_fixed_class_detector_lab_api.py index 6c9155c..0bc452e 100644 --- a/src/k1link/web/m48s_fixed_class_detector_lab_api.py +++ b/src/k1link/web/m48s_fixed_class_detector_lab_api.py @@ -337,6 +337,10 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]: or method.get("completeness") != "complete" or not isinstance(metrics, dict) or (integrated and not isinstance(metrics.get("integrated_world_state"), dict)) + or ( + metrics.get("runtime_hardening") is not None + and not _valid_runtime_hardening(metrics["runtime_hardening"]) + ) or not isinstance(manifest.get("limitations"), list) or not isinstance(catalog_descriptor, dict) or catalog_descriptor.get("path") != "catalog.json" @@ -363,6 +367,63 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]: return {"manifest": manifest, "catalog": catalog} +def _valid_runtime_hardening(value: object) -> bool: + if not isinstance(value, dict) or value.get("schema_version") != ( + "missioncore.m48s-runtime-hardening-comparison/v1" + ): + return False + full_run_keys = { + "source_frames_admitted", + "delivered_world_states", + "superseded_frames", + "effective_world_state_fps", + "world_state_completion_age_p95_ms", + "world_state_completion_age_p99_ms", + "world_state_completion_age_maximum_ms", + "rolling_maximum_ms", + "geometry_maximum_ms", + "additional_inference_passes", + } + for name in ("baseline", "hardened"): + run = value.get(name) + if ( + not isinstance(run, dict) + or set(run) != full_run_keys + or any(not _nonnegative_number(item) for item in run.values()) + ): + return False + startup = value.get("startup") + if not isinstance(startup, dict) or set(startup) != { + "baseline", + "prewarmed", + "prewarm_duration_ms", + "prewarm_inference_passes", + "validation_frames", + }: + return False + for name in ("baseline", "prewarmed"): + frame = startup.get(name) + if ( + not isinstance(frame, dict) + or set(frame) != {"detector_ms", "world_state_ms"} + or any(not _nonnegative_number(item) for item in frame.values()) + ): + return False + return ( + _nonnegative_number(startup.get("prewarm_duration_ms")) + and isinstance(startup.get("prewarm_inference_passes"), int) + and not isinstance(startup.get("prewarm_inference_passes"), bool) + and startup["prewarm_inference_passes"] > 0 + and isinstance(startup.get("validation_frames"), int) + and not isinstance(startup.get("validation_frames"), bool) + and startup["validation_frames"] > 0 + ) + + +def _nonnegative_number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and float(value) >= 0.0 + + def _candidate_signature(candidate: Path) -> tuple[int, ...]: if not candidate.is_dir() or candidate.is_symlink(): raise RuntimeError("M4.8S result candidate is invalid") diff --git a/tests/test_m48s_fixed_class_detector_lab.py b/tests/test_m48s_fixed_class_detector_lab.py index 72eba14..94f68f2 100644 --- a/tests/test_m48s_fixed_class_detector_lab.py +++ b/tests/test_m48s_fixed_class_detector_lab.py @@ -38,7 +38,7 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N assert manifest["result_id"] == result.result_id assert result.result_id.endswith(identity_digest) assert manifest["identity_sha256"] == identity_digest - assert len(manifest["artifacts"]) == 31 + assert len(manifest["artifacts"]) == 35 assert manifest["method"]["completeness"] == "complete" assert manifest["bounded_question_accepted"] is True assert manifest["ground_truth"] is False @@ -66,6 +66,20 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N assert max(integrated["queue_high_watermarks"].values()) <= 2 assert integrated["additional_inference_passes"] == 0 assert integrated["failures"] == 0 + hardening = manifest["metrics"]["runtime_hardening"] + assert hardening["schema_version"] == "missioncore.m48s-runtime-hardening-comparison/v1" + assert hardening["baseline"]["delivered_world_states"] == 4_480 + assert hardening["baseline"]["superseded_frames"] == 9 + assert hardening["hardened"]["delivered_world_states"] == 4_488 + assert hardening["hardened"]["superseded_frames"] == 1 + assert hardening["baseline"]["rolling_maximum_ms"] == 762.638263 + assert hardening["hardened"]["rolling_maximum_ms"] == 28.044958 + assert hardening["startup"]["baseline"]["detector_ms"] == 397.356888 + assert hardening["startup"]["prewarmed"]["detector_ms"] == 27.662887 + assert hardening["startup"]["baseline"]["world_state_ms"] == 449.893287 + assert hardening["startup"]["prewarmed"]["world_state_ms"] == 60.516957 + assert hardening["startup"]["prewarm_duration_ms"] == 420.721918 + assert hardening["startup"]["validation_frames"] == 1_000 catalog = json.loads((result.result_root / "catalog.json").read_text("utf-8")) assert catalog["frame_count"] == len(FRAME_IDS) == 11 @@ -85,7 +99,7 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N ) proof = verify_laboratory_evidence_result(definition, result.result_root) assert proof["result_id"] == result.result_id - assert proof["artifact_count"] == 31 + assert proof["artifact_count"] == 35 with pytest.raises(M48SFixedClassDetectorLabError, match="already exists"): build_m48s_fixed_class_detector_lab( diff --git a/tests/test_m48s_fixed_class_detector_lab_api.py b/tests/test_m48s_fixed_class_detector_lab_api.py index db32dd5..46059ba 100644 --- a/tests/test_m48s_fixed_class_detector_lab_api.py +++ b/tests/test_m48s_fixed_class_detector_lab_api.py @@ -47,20 +47,19 @@ def test_m48s_lab_api_projects_verified_result_frame_and_camera(tmp_path: Path) result.json()["metrics"]["integrated_world_state"]["world_state_completion_age_p95_ms"] == 74.733648 ) + hardening = result.json()["metrics"]["runtime_hardening"] + assert hardening["baseline"]["delivered_world_states"] == 4_480 + assert hardening["hardened"]["delivered_world_states"] == 4_488 + assert hardening["startup"]["prewarmed"]["world_state_ms"] == 60.516957 assert result.json()["ground_truth"] is False assert len(result.json()["frames"]) == 11 - timeline = client.get( - f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/timeline" - ) + timeline = client.get(f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/timeline") assert timeline.status_code == 200 assert timeline.json()["frame_count"] == 4_489 assert timeline.json()["world_state_frame_count"] == 4_481 assert timeline.json()["superseded_frame_count"] == 8 - assert ( - timeline.json()["camera_point_delivery"] - == "factory-kb4-causal-registered-accumulation" - ) + assert timeline.json()["camera_point_delivery"] == "factory-kb4-causal-registered-accumulation" assert timeline.json()["camera_point_window_seconds"] == 2.0 assert timeline.json()["camera_point_sample_limit"] == 20_000 chunk = client.get( @@ -72,9 +71,7 @@ def test_m48s_lab_api_projects_verified_result_frame_and_camera(tmp_path: Path) assert replay_frame["world_state_available"] is True assert replay_frame["camera_projection"] == "factory-kb4-exact" assert replay_frame["camera_projected_sample_count"] > 0 - assert any( - item["semantic_hint"] == "dog" for item in replay_frame["camera_proposals"] - ) + assert any(item["semantic_hint"] == "dog" for item in replay_frame["camera_proposals"]) camera_points = client.get( f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}" "/timeline/frames/253/camera-points" @@ -82,9 +79,7 @@ def test_m48s_lab_api_projects_verified_result_frame_and_camera(tmp_path: Path) assert camera_points.status_code == 200 camera_point_payload = camera_points.json() assert camera_point_payload["schema_version"] == "missioncore.m48s-camera-point-overlay/v1" - assert camera_point_payload["projection"] == ( - "factory-kb4-causal-registered-accumulation" - ) + assert camera_point_payload["projection"] == ("factory-kb4-causal-registered-accumulation") assert camera_point_payload["source_frame_count"] > 1 assert camera_point_payload["sample_count"] > replay_frame["camera_projected_sample_count"] assert camera_point_payload["sample_count"] <= 20_000