feat(laboratory): publish repeated M48S load envelope
This commit is contained in:
DCCONSTRUCTIONS
@@ -22,7 +22,7 @@ CATALOG_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame-catalog/v1"
|
||||
FRAME_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame/v1"
|
||||
REPORT_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-report/v1"
|
||||
RUNTIME_HARDENING_SCHEMA: Final = "missioncore.m48s-runtime-hardening-comparison/v1"
|
||||
LOAD_ENVELOPE_SCHEMA: Final = "missioncore.m48s-load-envelope-comparison/v1"
|
||||
LOAD_ENVELOPE_SCHEMA: Final = "missioncore.m48s-load-envelope-comparison/v2"
|
||||
METHOD_SCHEMA: Final = "missioncore.laboratory-method/v1"
|
||||
RESULT_PREFIX: Final = "m48s-fixed-class-detector-lab-"
|
||||
TOURNAMENT_ID: Final = (
|
||||
@@ -67,45 +67,123 @@ RUNTIME_HARDENING_RUNS: Final = {
|
||||
},
|
||||
}
|
||||
LOAD_ENVELOPE_RUNS: Final = {
|
||||
"production-10fps": {
|
||||
"run_id": "m48s-load-envelope-v1-production-10fps-a1",
|
||||
"production-10fps-a1": {
|
||||
"scenario_id": "production-10fps",
|
||||
"repetition": 1,
|
||||
"run_id": "m48s-load-prefetch-v1-production-10fps-a1",
|
||||
"load_purpose": "production-rate",
|
||||
"source_rate_hz": 10.0,
|
||||
"result_sha256": "60d4d82a8d47e5087019640353c9f8f7ddae518b52801dfd8c64423f5aba9fe2",
|
||||
"frames_sha256": "9e836f13a49390be498ce2b133fe2c20538af43adf667b0e7b89dc6349379ab0",
|
||||
"delivered": 4483,
|
||||
"superseded": 6,
|
||||
"operating_target_gate_passed": False,
|
||||
},
|
||||
"reserve-12fps": {
|
||||
"run_id": "m48s-load-envelope-v1-reserve-12fps-a1",
|
||||
"load_purpose": "reserve-gate",
|
||||
"source_rate_hz": 12.0,
|
||||
"result_sha256": "c8cb917f6f3a8cb6ffcc5146211066dbaed4aa2d8671e55820c266c49d5d5321",
|
||||
"frames_sha256": "3c1e635f6fbea643024bbd1711fb6c1ed5380430972cbb434ec76f4488d1c8b7",
|
||||
"delivered": 4477,
|
||||
"superseded": 12,
|
||||
"result_sha256": "0cf8b682b3972d434e9e10a6eb68a5abc2c1005d731cd61374de951f13d45282",
|
||||
"frames_sha256": "40d02cb59cb3531b764eface2e5f8b8dbf1d1e4ac83801746a97306ac8363910",
|
||||
"delivered": 4489,
|
||||
"superseded": 0,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"limit-15fps": {
|
||||
"run_id": "m48s-load-envelope-v1-limit-15fps-a1",
|
||||
"load_purpose": "limit-discovery",
|
||||
"source_rate_hz": 15.0,
|
||||
"result_sha256": "a78767ee2a7d8f3f7f7625c408e9bc8fb6c60e8ba6f77ab54fddbca018540713",
|
||||
"frames_sha256": "8be16ed155111ffa7093bbf46ceda4bc679593bfaf7a00c38c5d50c2385138d0",
|
||||
"reserve-12fps-a1": {
|
||||
"scenario_id": "reserve-12fps",
|
||||
"repetition": 1,
|
||||
"run_id": "m48s-load-prefetch-v1-reserve-12fps-a1",
|
||||
"load_purpose": "reserve-gate",
|
||||
"source_rate_hz": 12.0,
|
||||
"result_sha256": "70e0a623624ed162387960867d3615139e452ff30a6ba8daebe6965f39d76f58",
|
||||
"frames_sha256": "d54187c1b56e43f06498d2deac26374120a36cfd113c3b8f3e4dc170280353aa",
|
||||
"delivered": 4488,
|
||||
"superseded": 1,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"limit-15fps-a1": {
|
||||
"scenario_id": "limit-15fps",
|
||||
"repetition": 1,
|
||||
"run_id": "m48s-load-prefetch-v1-limit-15fps-a1",
|
||||
"load_purpose": "limit-discovery",
|
||||
"source_rate_hz": 15.0,
|
||||
"result_sha256": "d97ebdcc3bd70dc50284c6482fef614b7ec76180749c32b45c31d5e7e6667133",
|
||||
"frames_sha256": "b1b79205d266585b0d545bb5a02e87c957759774aeacab2f2d5290cb5d735bf7",
|
||||
"delivered": 4488,
|
||||
"superseded": 1,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"production-10fps-a2": {
|
||||
"scenario_id": "production-10fps",
|
||||
"repetition": 2,
|
||||
"run_id": "m48s-load-prefetch-v1-production-10fps-a2",
|
||||
"load_purpose": "production-rate",
|
||||
"source_rate_hz": 10.0,
|
||||
"result_sha256": "c98c052b759cecd390e6e79d9eac223e45c96e532bd974d9961d5904945fbfe8",
|
||||
"frames_sha256": "8574b98aee54ea25e31fc80cbcb7720683bdb36116dc3f2606aed4b0316b8cfa",
|
||||
"delivered": 4489,
|
||||
"superseded": 0,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"reserve-12fps-a2": {
|
||||
"scenario_id": "reserve-12fps",
|
||||
"repetition": 2,
|
||||
"run_id": "m48s-load-prefetch-v1-reserve-12fps-a2",
|
||||
"load_purpose": "reserve-gate",
|
||||
"source_rate_hz": 12.0,
|
||||
"result_sha256": "16cc2cb0fd9a56707910dfb397021613eb57b1a1d6259b52d14ec38a1e348eb6",
|
||||
"frames_sha256": "cabd11a2a363b702e2a345198de12408079532ebaa8a095bcfa458acf763325f",
|
||||
"delivered": 4489,
|
||||
"superseded": 0,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"limit-15fps-a2": {
|
||||
"scenario_id": "limit-15fps",
|
||||
"repetition": 2,
|
||||
"run_id": "m48s-load-prefetch-v1-limit-15fps-a2",
|
||||
"load_purpose": "limit-discovery",
|
||||
"source_rate_hz": 15.0,
|
||||
"result_sha256": "bf23b09b6874f506170448484d1ca2886d8abeb951f9536e810ed24991a0efa6",
|
||||
"frames_sha256": "fe3b37bde43fe92d92e5da4004509ef87aa85b25700cf8d9d5c84817e13b2422",
|
||||
"delivered": 4488,
|
||||
"superseded": 1,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"production-10fps-a3": {
|
||||
"scenario_id": "production-10fps",
|
||||
"repetition": 3,
|
||||
"run_id": "m48s-load-prefetch-v1-production-10fps-a3",
|
||||
"load_purpose": "production-rate",
|
||||
"source_rate_hz": 10.0,
|
||||
"result_sha256": "ebb81d69c16b32aa6f2237a3a5e183def58dcb422b1b82fa2621d340cc1b6fcc",
|
||||
"frames_sha256": "2668b936d274a273dbe440dade55716f686af507aaa4f1a02803396accfe240d",
|
||||
"delivered": 4489,
|
||||
"superseded": 0,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"reserve-12fps-a3": {
|
||||
"scenario_id": "reserve-12fps",
|
||||
"repetition": 3,
|
||||
"run_id": "m48s-load-prefetch-v1-reserve-12fps-a3",
|
||||
"load_purpose": "reserve-gate",
|
||||
"source_rate_hz": 12.0,
|
||||
"result_sha256": "655d3c0b6c7210f12cfb3a9edf84da5851459afa7303c9732b879e454ac05dc2",
|
||||
"frames_sha256": "419e8862f0da0e54a3b82f581230ab72874bf52841f62a4e49e1905ed5a77520",
|
||||
"delivered": 4485,
|
||||
"superseded": 4,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"limit-15fps-a3": {
|
||||
"scenario_id": "limit-15fps",
|
||||
"repetition": 3,
|
||||
"run_id": "m48s-load-prefetch-v1-limit-15fps-a3",
|
||||
"load_purpose": "limit-discovery",
|
||||
"source_rate_hz": 15.0,
|
||||
"result_sha256": "e844183dcee5982f389c9aa847cd98cec78fb81e0b3920e606063740eb35f421",
|
||||
"frames_sha256": "477cffcecbaf607c8c68979b5425f84963d07f1be76917d2a21925f290bb2736",
|
||||
"delivered": 4489,
|
||||
"superseded": 0,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
}
|
||||
LOAD_ENVELOPE_PROFILE_SHA256: Final = (
|
||||
"d63c711534d84ee526d056885cfa0b5221df61d8394b244188b735bb34d20869"
|
||||
"f0c9467d71622085b31f650d89b70ad5107fa49842f1bc828ccc2c83f85628a3"
|
||||
)
|
||||
LOAD_RUNTIME_ARTIFACT_SHA256: Final = (
|
||||
"c2344c91ce48111abb364cad38bd73fca9e5b30c8d001c113718dc1b30a8d859"
|
||||
"9929b735fd139369812f2d5c67149f7737cbf3d3f34076b590e9e9672666bf03"
|
||||
)
|
||||
LOAD_RUNNER_SHA256: Final = (
|
||||
"0ff2ed9bdf5d5982cce08c4930853f65aa7be2c5986afacfe6a7cf925a2815e9"
|
||||
"d3ca51b8500b681307d3a9974f57ae72cdbf04267b5bba26e7ffb091ddd9df31"
|
||||
)
|
||||
YOLOX_ID: Final = (
|
||||
"m48s-yolox-all-coco-shadow-7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06"
|
||||
@@ -191,9 +269,14 @@ def build_m48s_fixed_class_detector_lab(
|
||||
}
|
||||
for name, definition in RUNTIME_HARDENING_RUNS.items()
|
||||
}
|
||||
load_envelope_profile_path = repository / "config/perception/m48s-load-envelope-v1.json"
|
||||
load_envelope_profile_path = (
|
||||
repository / "config/perception/m48s-load-envelope-prefetch-v1.json"
|
||||
)
|
||||
load_envelope_paths = {
|
||||
name: runtime / "load-envelope-worker" / str(definition["run_id"]) / "result.json"
|
||||
name: runtime
|
||||
/ "load-envelope-prefetch-worker"
|
||||
/ str(definition["run_id"])
|
||||
/ "result.json"
|
||||
for name, definition in LOAD_ENVELOPE_RUNS.items()
|
||||
}
|
||||
yolox_root = runtime / "yolox-all-coco-results" / YOLOX_ID
|
||||
@@ -352,7 +435,7 @@ def build_m48s_fixed_class_detector_lab(
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||
result_id = RESULT_PREFIX + identity_sha256
|
||||
completed_utc_ns = load_envelope_runs["limit-15fps"].get("completed_utc_ns")
|
||||
completed_utc_ns = load_envelope_runs["limit-15fps-a3"].get("completed_utc_ns")
|
||||
if not isinstance(completed_utc_ns, int) or isinstance(completed_utc_ns, bool):
|
||||
raise M48SFixedClassDetectorLabError(
|
||||
"complete reference-graph completion time is unavailable"
|
||||
@@ -378,6 +461,7 @@ def build_m48s_fixed_class_detector_lab(
|
||||
hardening_first_frames=hardening_first_frames,
|
||||
load_envelope_runs=load_envelope_runs,
|
||||
)
|
||||
envelope = cast(dict[str, Any], metrics["load_envelope"])
|
||||
decision = {
|
||||
"bounded_question_accepted": True,
|
||||
"selected_candidate": "rf-detr",
|
||||
@@ -386,10 +470,12 @@ def build_m48s_fixed_class_detector_lab(
|
||||
"integrated_world_state_gate_passed": True,
|
||||
"full_replay_visual_published": True,
|
||||
"load_envelope_evaluated": True,
|
||||
"production_rate_repeatability_passed": False,
|
||||
"reserve_12_fps_passed": True,
|
||||
"limit_15_fps_passed": True,
|
||||
"load_envelope_accepted": False,
|
||||
"production_rate_repeatability_passed": envelope[
|
||||
"production_rate_repeatability_passed"
|
||||
],
|
||||
"reserve_12_fps_passed": envelope["reserve_12_fps_passed"],
|
||||
"limit_15_fps_passed": envelope["limit_15_fps_passed"],
|
||||
"load_envelope_accepted": envelope["load_envelope_accepted"],
|
||||
"detector_replacement_authorized": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
@@ -409,9 +495,9 @@ def build_m48s_fixed_class_detector_lab(
|
||||
"metrics come from separately sealed, input-identical replay runs."
|
||||
),
|
||||
(
|
||||
"The 12 and 15 FPS operating gates passed, but the separately predeclared 10 FPS "
|
||||
"delivery-ratio gate missed by two frames; capacity is demonstrated while delivery "
|
||||
"repeatability remains open."
|
||||
"Three complete repetitions at each of 10, 12 and 15 FPS passed their predeclared "
|
||||
"integrity and operating-target gates after bounded decode prefetch was completed "
|
||||
"before source admission."
|
||||
),
|
||||
]
|
||||
|
||||
@@ -763,23 +849,38 @@ def _validate_load_envelope(
|
||||
) -> None:
|
||||
scenarios = profile.get("scenarios")
|
||||
if (
|
||||
profile.get("schema_version") != "missioncore.m48s-load-envelope-profile/v1"
|
||||
or profile.get("profile_id") != "m48s-rf-detr-reference-graph-load-envelope/v1"
|
||||
profile.get("schema_version") != "missioncore.m48s-load-envelope-profile/v2"
|
||||
or profile.get("profile_id")
|
||||
!= "m48s-rf-detr-reference-graph-load-envelope-prefetch/v1"
|
||||
or sha256_path(profile_path) != LOAD_ENVELOPE_PROFILE_SHA256
|
||||
or not isinstance(scenarios, list)
|
||||
or len(scenarios) != len(LOAD_ENVELOPE_RUNS)
|
||||
or len(scenarios) != 3
|
||||
or profile.get("series_control", {}).get("repeat_count_per_rate") != 3
|
||||
):
|
||||
raise M48SFixedClassDetectorLabError("load-envelope profile identity changed")
|
||||
scenarios_by_id = {
|
||||
scenario.get("id"): scenario for scenario in scenarios if isinstance(scenario, dict)
|
||||
}
|
||||
if set(scenarios_by_id) != set(LOAD_ENVELOPE_RUNS):
|
||||
expected_scenario_ids = {
|
||||
str(definition["scenario_id"]) for definition in LOAD_ENVELOPE_RUNS.values()
|
||||
}
|
||||
if set(scenarios_by_id) != expected_scenario_ids:
|
||||
raise M48SFixedClassDetectorLabError("load-envelope scenarios changed")
|
||||
declared_run_ids = {
|
||||
run_id
|
||||
for scenario in scenarios_by_id.values()
|
||||
for run_id in cast(list[object], scenario.get("run_ids", []))
|
||||
}
|
||||
expected_run_ids = {
|
||||
str(definition["run_id"]) for definition in LOAD_ENVELOPE_RUNS.values()
|
||||
}
|
||||
if declared_run_ids != expected_run_ids:
|
||||
raise M48SFixedClassDetectorLabError("load-envelope run series changed")
|
||||
shared_inputs: dict[str, object] | None = None
|
||||
for name, definition in LOAD_ENVELOPE_RUNS.items():
|
||||
run = runs.get(name)
|
||||
path = paths.get(name)
|
||||
scenario = scenarios_by_id[name]
|
||||
scenario = scenarios_by_id[str(definition["scenario_id"])]
|
||||
if not isinstance(run, dict) or not isinstance(path, Path):
|
||||
raise M48SFixedClassDetectorLabError("load-envelope evidence is incomplete")
|
||||
source = run.get("source")
|
||||
@@ -790,8 +891,22 @@ def _validate_load_envelope(
|
||||
frame_evidence = execution.get("frame_evidence") if isinstance(execution, dict) else None
|
||||
terminal = execution.get("terminal_outcomes") if isinstance(execution, dict) else None
|
||||
pipeline = run.get("metrics", {}).get("pipeline_timing")
|
||||
decode_by_phase = (
|
||||
pipeline.get("decode_duration_by_phase_ms") if isinstance(pipeline, dict) else None
|
||||
)
|
||||
pacing = (
|
||||
pipeline.get("delivered_source_pacing_lateness_ms")
|
||||
if isinstance(pipeline, dict)
|
||||
else None
|
||||
)
|
||||
loops = execution.get("loops") if isinstance(execution, dict) else None
|
||||
prefetch = (
|
||||
loops[0].get("source_prefetch")
|
||||
if isinstance(loops, list) and len(loops) == 1 and isinstance(loops[0], dict)
|
||||
else None
|
||||
)
|
||||
if (
|
||||
run.get("schema_version") != "missioncore.m48s-reference-graph-shadow-load/v4"
|
||||
run.get("schema_version") != "missioncore.m48s-reference-graph-shadow-load/v5"
|
||||
or run.get("completed") is not True
|
||||
or run.get("production_accepted") is not False
|
||||
or run.get("authority") != false_authority()
|
||||
@@ -821,14 +936,27 @@ def _validate_load_envelope(
|
||||
or execution.get("delivered_world_states") != definition["delivered"]
|
||||
or not isinstance(terminal, dict)
|
||||
or terminal.get("delivered") != definition["delivered"]
|
||||
or terminal.get("superseded") != definition["superseded"]
|
||||
or terminal.get("superseded", 0) != definition["superseded"]
|
||||
or sum(terminal.values()) != 4489
|
||||
or not isinstance(frame_evidence, dict)
|
||||
or frame_evidence.get("row_count") != definition["delivered"]
|
||||
or frame_evidence.get("sha256") != definition["frames_sha256"]
|
||||
or not isinstance(pipeline, dict)
|
||||
or pipeline.get("additional_inference_passes") != 0
|
||||
or not isinstance(decode_by_phase, dict)
|
||||
or set(decode_by_phase) != {"preadmission", "hot-loop"}
|
||||
or any(
|
||||
not isinstance(decode_by_phase.get(phase), dict)
|
||||
for phase in ("preadmission", "hot-loop")
|
||||
)
|
||||
or not isinstance(pacing, dict)
|
||||
or not isinstance(prefetch, dict)
|
||||
or prefetch.get("capacity_frames") != 64
|
||||
or prefetch.get("ready_frames") != 64
|
||||
or prefetch.get("buffered_frames") != 64
|
||||
or not isinstance(prefetch.get("preparation_duration_ns"), int)
|
||||
or thresholds != scenario.get("thresholds")
|
||||
or definition["run_id"] not in scenario.get("run_ids", [])
|
||||
or sha256_path(path) != definition["result_sha256"]
|
||||
):
|
||||
raise M48SFixedClassDetectorLabError(
|
||||
@@ -1054,54 +1182,164 @@ def _metrics(
|
||||
|
||||
def _load_envelope_metrics(runs: dict[str, dict[str, Any]]) -> dict[str, object]:
|
||||
scenarios: list[dict[str, object]] = []
|
||||
for name in LOAD_ENVELOPE_RUNS:
|
||||
run = runs[name]
|
||||
execution = run["execution"]
|
||||
metrics = run["metrics"]
|
||||
completion = metrics["world_state_completion_age_ms"]
|
||||
gpu = metrics["gpu"]
|
||||
pipeline = metrics["pipeline_timing"]
|
||||
terminal = execution["terminal_outcomes"]
|
||||
thresholds = run["predeclared_thresholds"]
|
||||
scenario_order = ("production-10fps", "reserve-12fps", "limit-15fps")
|
||||
for scenario_id in scenario_order:
|
||||
definitions = sorted(
|
||||
(
|
||||
(name, definition)
|
||||
for name, definition in LOAD_ENVELOPE_RUNS.items()
|
||||
if definition["scenario_id"] == scenario_id
|
||||
),
|
||||
key=lambda item: int(item[1]["repetition"]),
|
||||
)
|
||||
repetitions: list[dict[str, object]] = []
|
||||
queue_high_watermarks: dict[str, int] = {}
|
||||
for name, definition in definitions:
|
||||
run = runs[name]
|
||||
execution = run["execution"]
|
||||
metrics = run["metrics"]
|
||||
completion = metrics["world_state_completion_age_ms"]
|
||||
pipeline = metrics["pipeline_timing"]
|
||||
decode = pipeline["decode_duration_by_phase_ms"]
|
||||
pacing = pipeline["delivered_source_pacing_lateness_ms"]
|
||||
terminal = execution["terminal_outcomes"]
|
||||
prefetch = execution["loops"][0]["source_prefetch"]
|
||||
for queue, depth in execution["queue_high_watermarks"].items():
|
||||
queue_high_watermarks[queue] = max(
|
||||
queue_high_watermarks.get(queue, 0), int(depth)
|
||||
)
|
||||
repetitions.append(
|
||||
{
|
||||
"run_id": definition["run_id"],
|
||||
"repetition": definition["repetition"],
|
||||
"delivered_world_states": execution["delivered_world_states"],
|
||||
"superseded_frames": terminal.get("superseded", 0),
|
||||
"delivery_ratio": execution["delivery_ratio"],
|
||||
"effective_world_state_fps": execution["effective_world_state_fps"],
|
||||
"world_state_completion_age_p95_ms": completion["p95"],
|
||||
"hot_loop_decode_p95_ms": decode["hot-loop"]["p95"],
|
||||
"hot_loop_decode_maximum_ms": decode["hot-loop"]["maximum"],
|
||||
"preadmission_decode_maximum_ms": decode["preadmission"]["maximum"],
|
||||
"source_pacing_lateness_maximum_ms": pacing["maximum"],
|
||||
"source_prefetch_preparation_ms": (
|
||||
prefetch["preparation_duration_ns"] / 1_000_000
|
||||
),
|
||||
"operating_target_gate_passed": run["operating_target_gate_passed"],
|
||||
"result_sha256": definition["result_sha256"],
|
||||
"frame_evidence_sha256": execution["frame_evidence"]["sha256"],
|
||||
}
|
||||
)
|
||||
first_name = definitions[0][0]
|
||||
first_run = runs[first_name]
|
||||
run_values = [runs[name] for name, _definition in definitions]
|
||||
executions = [run["execution"] for run in run_values]
|
||||
run_metrics = [run["metrics"] for run in run_values]
|
||||
pipelines = [metrics["pipeline_timing"] for metrics in run_metrics]
|
||||
completions = [metrics["world_state_completion_age_ms"] for metrics in run_metrics]
|
||||
gpus = [metrics["gpu"] for metrics in run_metrics]
|
||||
all_integrity = all(run["evidence_integrity_gate_passed"] for run in run_values)
|
||||
all_targets = all(run["operating_target_gate_passed"] for run in run_values)
|
||||
scenarios.append(
|
||||
{
|
||||
"id": name,
|
||||
"load_purpose": execution["load_purpose"],
|
||||
"requested_source_rate_hz": execution["requested_source_rate_hz"],
|
||||
"source_frames_admitted": execution["admitted_frames"],
|
||||
"delivered_world_states": execution["delivered_world_states"],
|
||||
"superseded_frames": terminal.get("superseded", 0),
|
||||
"delivery_ratio": execution["delivery_ratio"],
|
||||
"effective_world_state_fps": execution["effective_world_state_fps"],
|
||||
"world_state_completion_age_p95_ms": completion["p95"],
|
||||
"world_state_completion_age_p99_ms": completion["p99"],
|
||||
"world_state_completion_age_maximum_ms": completion["maximum"],
|
||||
"decode_p95_ms": pipeline["decode_duration_ms"]["p95"],
|
||||
"decode_maximum_ms": pipeline["decode_duration_ms"]["maximum"],
|
||||
"detector_p95_ms": pipeline["detector_ms"]["total"]["p95"],
|
||||
"detector_maximum_ms": pipeline["detector_ms"]["total"]["maximum"],
|
||||
"gpu_utilization_mean_percent": gpu["gpu_utilization_percent"]["mean"],
|
||||
"gpu_utilization_maximum_percent": gpu["gpu_utilization_percent"]["maximum"],
|
||||
"gpu_memory_maximum_mib": gpu["gpu_memory_used_mib"]["maximum"],
|
||||
"process_peak_rss_mib": metrics["process_peak_rss_after_mib"],
|
||||
"queue_high_watermarks": execution["queue_high_watermarks"],
|
||||
"id": scenario_id,
|
||||
"load_purpose": first_run["execution"]["load_purpose"],
|
||||
"requested_source_rate_hz": first_run["execution"][
|
||||
"requested_source_rate_hz"
|
||||
],
|
||||
"repeat_count": len(repetitions),
|
||||
"all_repetitions_passed": all_integrity and all_targets,
|
||||
"source_frames_admitted": min(
|
||||
execution["admitted_frames"] for execution in executions
|
||||
),
|
||||
"delivered_world_states": min(
|
||||
execution["delivered_world_states"] for execution in executions
|
||||
),
|
||||
"superseded_frames": max(
|
||||
execution["terminal_outcomes"].get("superseded", 0)
|
||||
for execution in executions
|
||||
),
|
||||
"delivery_ratio": min(execution["delivery_ratio"] for execution in executions),
|
||||
"effective_world_state_fps": min(
|
||||
execution["effective_world_state_fps"] for execution in executions
|
||||
),
|
||||
"world_state_completion_age_p95_ms": max(
|
||||
completion["p95"] for completion in completions
|
||||
),
|
||||
"world_state_completion_age_p99_ms": max(
|
||||
completion["p99"] for completion in completions
|
||||
),
|
||||
"world_state_completion_age_maximum_ms": max(
|
||||
completion["maximum"] for completion in completions
|
||||
),
|
||||
"hot_loop_decode_p95_ms": max(
|
||||
pipeline["decode_duration_by_phase_ms"]["hot-loop"]["p95"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"hot_loop_decode_maximum_ms": max(
|
||||
pipeline["decode_duration_by_phase_ms"]["hot-loop"]["maximum"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"preadmission_decode_maximum_ms": max(
|
||||
pipeline["decode_duration_by_phase_ms"]["preadmission"]["maximum"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"source_prefetch_preparation_maximum_ms": max(
|
||||
repetition["source_prefetch_preparation_ms"]
|
||||
for repetition in repetitions
|
||||
),
|
||||
"source_pacing_lateness_p95_ms": max(
|
||||
pipeline["delivered_source_pacing_lateness_ms"]["p95"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"source_pacing_lateness_maximum_ms": max(
|
||||
pipeline["delivered_source_pacing_lateness_ms"]["maximum"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"detector_p95_ms": max(
|
||||
pipeline["detector_ms"]["total"]["p95"] for pipeline in pipelines
|
||||
),
|
||||
"detector_maximum_ms": max(
|
||||
pipeline["detector_ms"]["total"]["maximum"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"gpu_utilization_mean_percent": max(
|
||||
gpu["gpu_utilization_percent"]["mean"] for gpu in gpus
|
||||
),
|
||||
"gpu_utilization_maximum_percent": max(
|
||||
gpu["gpu_utilization_percent"]["maximum"] for gpu in gpus
|
||||
),
|
||||
"gpu_memory_maximum_mib": max(
|
||||
gpu["gpu_memory_used_mib"]["maximum"] for gpu in gpus
|
||||
),
|
||||
"process_peak_rss_mib": max(
|
||||
metrics["process_peak_rss_after_mib"] for metrics in run_metrics
|
||||
),
|
||||
"queue_high_watermarks": queue_high_watermarks,
|
||||
"queue_capacity": 2,
|
||||
"additional_inference_passes": pipeline["additional_inference_passes"],
|
||||
"integrity_gate_passed": run["evidence_integrity_gate_passed"],
|
||||
"operating_target_gate_passed": run["operating_target_gate_passed"],
|
||||
"thresholds": thresholds,
|
||||
"frame_evidence_sha256": execution["frame_evidence"]["sha256"],
|
||||
"additional_inference_passes": max(
|
||||
pipeline["additional_inference_passes"] for pipeline in pipelines
|
||||
),
|
||||
"integrity_gate_passed": all_integrity,
|
||||
"operating_target_gate_passed": all_targets,
|
||||
"thresholds": first_run["predeclared_thresholds"],
|
||||
"repetitions": repetitions,
|
||||
}
|
||||
)
|
||||
production_passed = cast(bool, scenarios[0]["all_repetitions_passed"])
|
||||
reserve_passed = cast(bool, scenarios[1]["all_repetitions_passed"])
|
||||
limit_passed = cast(bool, scenarios[2]["all_repetitions_passed"])
|
||||
return {
|
||||
"schema_version": LOAD_ENVELOPE_SCHEMA,
|
||||
"production_rate_repeatability_passed": False,
|
||||
"reserve_12_fps_passed": True,
|
||||
"limit_15_fps_passed": True,
|
||||
"load_envelope_accepted": False,
|
||||
"compute_capacity_at_least_fps": 15.0,
|
||||
"repeat_count_per_rate": 3,
|
||||
"production_rate_repeatability_passed": production_passed,
|
||||
"reserve_12_fps_passed": reserve_passed,
|
||||
"limit_15_fps_passed": limit_passed,
|
||||
"load_envelope_accepted": production_passed and reserve_passed,
|
||||
"compute_capacity_at_least_fps": (
|
||||
15.0 if limit_passed else 12.0 if reserve_passed else 10.0
|
||||
),
|
||||
"bottleneck_interpretation": (
|
||||
"rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation"
|
||||
"cold-video-decode-isolated-before-admission-no-steady-gpu-saturation"
|
||||
),
|
||||
"scenarios": scenarios,
|
||||
}
|
||||
@@ -1261,10 +1499,10 @@ def _artifact_manifest(root: Path) -> list[dict[str, object]]:
|
||||
)
|
||||
role = "upstream-runtime-hardening-evidence"
|
||||
elif relative == "runtime-load-envelope-profile.json":
|
||||
schema_version = "missioncore.m48s-load-envelope-profile/v1"
|
||||
schema_version = "missioncore.m48s-load-envelope-profile/v2"
|
||||
role = "predeclared-load-envelope-profile"
|
||||
elif relative.startswith("runtime-load-"):
|
||||
schema_version = "missioncore.m48s-reference-graph-shadow-load/v4"
|
||||
schema_version = "missioncore.m48s-reference-graph-shadow-load/v5"
|
||||
role = "upstream-load-envelope-evidence"
|
||||
artifacts.append(
|
||||
{
|
||||
|
||||
@@ -305,6 +305,7 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
|
||||
decision = manifest.get("decision")
|
||||
method = manifest.get("method")
|
||||
metrics = manifest.get("metrics")
|
||||
load_envelope = metrics.get("load_envelope") if isinstance(metrics, dict) else None
|
||||
status = manifest.get("status")
|
||||
integrated = status == INTEGRATED_STATUS
|
||||
if (
|
||||
@@ -332,13 +333,18 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
|
||||
or (integrated and decision.get("integrated_world_state_gate_passed") is not True)
|
||||
or (integrated and decision.get("detector_replacement_authorized") is not False)
|
||||
or (
|
||||
metrics.get("load_envelope") is not None
|
||||
load_envelope is not None
|
||||
and (
|
||||
decision.get("load_envelope_evaluated") is not True
|
||||
or decision.get("production_rate_repeatability_passed") is not False
|
||||
or decision.get("reserve_12_fps_passed") is not True
|
||||
or decision.get("limit_15_fps_passed") is not True
|
||||
or decision.get("load_envelope_accepted") is not False
|
||||
or not isinstance(load_envelope, dict)
|
||||
or decision.get("production_rate_repeatability_passed")
|
||||
is not load_envelope.get("production_rate_repeatability_passed")
|
||||
or decision.get("reserve_12_fps_passed")
|
||||
is not load_envelope.get("reserve_12_fps_passed")
|
||||
or decision.get("limit_15_fps_passed")
|
||||
is not load_envelope.get("limit_15_fps_passed")
|
||||
or decision.get("load_envelope_accepted")
|
||||
is not load_envelope.get("load_envelope_accepted")
|
||||
)
|
||||
)
|
||||
or decision.get("production_accepted") is not False
|
||||
@@ -435,10 +441,104 @@ def _valid_runtime_hardening(value: object) -> bool:
|
||||
|
||||
|
||||
def _valid_load_envelope(value: object) -> bool:
|
||||
if not isinstance(value, dict) or value.get("schema_version") != (
|
||||
"missioncore.m48s-load-envelope-comparison/v1"
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
if value.get("schema_version") == "missioncore.m48s-load-envelope-comparison/v1":
|
||||
return _valid_legacy_load_envelope(value)
|
||||
if value.get("schema_version") != "missioncore.m48s-load-envelope-comparison/v2":
|
||||
return False
|
||||
if (
|
||||
value.get("repeat_count_per_rate") != 3
|
||||
or value.get("production_rate_repeatability_passed") is not True
|
||||
or value.get("reserve_12_fps_passed") is not True
|
||||
or value.get("limit_15_fps_passed") is not True
|
||||
or value.get("load_envelope_accepted") is not True
|
||||
or value.get("compute_capacity_at_least_fps") != 15.0
|
||||
or value.get("bottleneck_interpretation")
|
||||
!= "cold-video-decode-isolated-before-admission-no-steady-gpu-saturation"
|
||||
):
|
||||
return False
|
||||
scenarios = value.get("scenarios")
|
||||
scenario_ids = [item.get("id") for item in scenarios if isinstance(item, dict)] if (
|
||||
isinstance(scenarios, list)
|
||||
) else []
|
||||
if scenario_ids != ["production-10fps", "reserve-12fps", "limit-15fps"]:
|
||||
return False
|
||||
required_numeric = {
|
||||
"requested_source_rate_hz",
|
||||
"source_frames_admitted",
|
||||
"delivered_world_states",
|
||||
"superseded_frames",
|
||||
"delivery_ratio",
|
||||
"effective_world_state_fps",
|
||||
"world_state_completion_age_p95_ms",
|
||||
"world_state_completion_age_p99_ms",
|
||||
"world_state_completion_age_maximum_ms",
|
||||
"hot_loop_decode_p95_ms",
|
||||
"hot_loop_decode_maximum_ms",
|
||||
"preadmission_decode_maximum_ms",
|
||||
"source_prefetch_preparation_maximum_ms",
|
||||
"source_pacing_lateness_p95_ms",
|
||||
"source_pacing_lateness_maximum_ms",
|
||||
"detector_p95_ms",
|
||||
"detector_maximum_ms",
|
||||
"gpu_utilization_mean_percent",
|
||||
"gpu_utilization_maximum_percent",
|
||||
"gpu_memory_maximum_mib",
|
||||
"process_peak_rss_mib",
|
||||
"queue_capacity",
|
||||
"additional_inference_passes",
|
||||
}
|
||||
for scenario in scenarios:
|
||||
repetitions = scenario.get("repetitions") if isinstance(scenario, dict) else None
|
||||
if (
|
||||
not isinstance(scenario, dict)
|
||||
or scenario.get("repeat_count") != 3
|
||||
or scenario.get("all_repetitions_passed") is not True
|
||||
or scenario.get("integrity_gate_passed") is not True
|
||||
or scenario.get("operating_target_gate_passed") is not True
|
||||
or scenario.get("additional_inference_passes") != 0
|
||||
or scenario.get("queue_capacity") != 2
|
||||
or not isinstance(scenario.get("queue_high_watermarks"), dict)
|
||||
or any(value != 2 for value in scenario["queue_high_watermarks"].values())
|
||||
or not isinstance(scenario.get("thresholds"), dict)
|
||||
or any(not _nonnegative_number(scenario.get(key)) for key in required_numeric)
|
||||
or not isinstance(repetitions, list)
|
||||
or len(repetitions) != 3
|
||||
):
|
||||
return False
|
||||
for expected_repetition, repetition in enumerate(repetitions, start=1):
|
||||
if (
|
||||
not isinstance(repetition, dict)
|
||||
or repetition.get("repetition") != expected_repetition
|
||||
or repetition.get("operating_target_gate_passed") is not True
|
||||
or not isinstance(repetition.get("run_id"), str)
|
||||
or not repetition["run_id"]
|
||||
or any(
|
||||
not _nonnegative_number(repetition.get(key))
|
||||
for key in {
|
||||
"delivered_world_states",
|
||||
"superseded_frames",
|
||||
"delivery_ratio",
|
||||
"effective_world_state_fps",
|
||||
"world_state_completion_age_p95_ms",
|
||||
"hot_loop_decode_p95_ms",
|
||||
"hot_loop_decode_maximum_ms",
|
||||
"preadmission_decode_maximum_ms",
|
||||
"source_pacing_lateness_maximum_ms",
|
||||
}
|
||||
)
|
||||
or any(
|
||||
not isinstance(repetition.get(key), str)
|
||||
or SHA256.fullmatch(repetition[key]) is None
|
||||
for key in ("result_sha256", "frame_evidence_sha256")
|
||||
)
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _valid_legacy_load_envelope(value: dict[str, object]) -> bool:
|
||||
if (
|
||||
value.get("production_rate_repeatability_passed") is not False
|
||||
or value.get("reserve_12_fps_passed") is not True
|
||||
@@ -450,10 +550,9 @@ def _valid_load_envelope(value: object) -> bool:
|
||||
):
|
||||
return False
|
||||
scenarios = value.get("scenarios")
|
||||
scenario_ids = [item.get("id") for item in scenarios if isinstance(item, dict)] if (
|
||||
isinstance(scenarios, list)
|
||||
) else []
|
||||
if scenario_ids != ["production-10fps", "reserve-12fps", "limit-15fps"]:
|
||||
if not isinstance(scenarios, list) or [
|
||||
item.get("id") for item in scenarios if isinstance(item, dict)
|
||||
] != ["production-10fps", "reserve-12fps", "limit-15fps"]:
|
||||
return False
|
||||
expected_targets = (False, True, True)
|
||||
required_numeric = {
|
||||
@@ -485,7 +584,7 @@ def _valid_load_envelope(value: object) -> bool:
|
||||
or scenario.get("additional_inference_passes") != 0
|
||||
or scenario.get("queue_capacity") != 2
|
||||
or not isinstance(scenario.get("queue_high_watermarks"), dict)
|
||||
or any(value != 2 for value in scenario["queue_high_watermarks"].values())
|
||||
or any(depth != 2 for depth in scenario["queue_high_watermarks"].values())
|
||||
or not isinstance(scenario.get("thresholds"), dict)
|
||||
or any(not _nonnegative_number(scenario.get(key)) for key in required_numeric)
|
||||
or not isinstance(scenario.get("frame_evidence_sha256"), str)
|
||||
|
||||
Reference in New Issue
Block a user