394 lines
15 KiB
JavaScript
394 lines
15 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { after, before, test } from "node:test";
|
|
import { createServer } from "vite";
|
|
|
|
let server;
|
|
let fetchM48SFixedClassDetectorResult;
|
|
let fetchM48SFixedClassDetectorFrame;
|
|
|
|
const resultId = `m48s-fixed-class-detector-lab-${"b".repeat(64)}`;
|
|
const authority = {
|
|
actuation_allowed: false,
|
|
candidate_accepted: false,
|
|
commands_enabled: false,
|
|
ground_truth: false,
|
|
navigation_or_safety_accepted: false,
|
|
};
|
|
|
|
before(async () => {
|
|
server = await createServer({
|
|
appType: "custom",
|
|
logLevel: "silent",
|
|
server: { middlewareMode: true },
|
|
});
|
|
({
|
|
fetchM48SFixedClassDetectorResult,
|
|
fetchM48SFixedClassDetectorFrame,
|
|
} = await server.ssrLoadModule("/src/core/laboratory/m48sFixedClassDetector.ts"));
|
|
});
|
|
|
|
after(async () => server?.close());
|
|
|
|
function response(value) {
|
|
return { ok: true, status: 200, json: async () => value };
|
|
}
|
|
|
|
function resultPayload() {
|
|
const candidate = (id, overrides = {}) => ({
|
|
id,
|
|
label: id,
|
|
provider_id: `${id}/v1`,
|
|
capacity_fps: 40,
|
|
p95_ms: 35,
|
|
frame_253_dog_detected: false,
|
|
frame_253_dog_score: null,
|
|
selected: false,
|
|
...overrides,
|
|
});
|
|
const loadScenario = (id, loadPurpose, rate, delivered, superseded) => ({
|
|
id,
|
|
load_purpose: loadPurpose,
|
|
requested_source_rate_hz: rate,
|
|
source_frames_admitted: 4489,
|
|
delivered_world_states: delivered,
|
|
superseded_frames: superseded,
|
|
delivery_ratio: delivered / 4489,
|
|
effective_world_state_fps: rate * 0.975,
|
|
world_state_completion_age_p95_ms: 75,
|
|
world_state_completion_age_p99_ms: 100,
|
|
world_state_completion_age_maximum_ms: 400,
|
|
hot_loop_decode_p95_ms: 14,
|
|
hot_loop_decode_maximum_ms: 42,
|
|
preadmission_decode_maximum_ms: 3633,
|
|
source_prefetch_preparation_maximum_ms: 4353,
|
|
source_pacing_lateness_p95_ms: 1.7,
|
|
source_pacing_lateness_maximum_ms: 11,
|
|
detector_p95_ms: 40,
|
|
detector_maximum_ms: 370,
|
|
gpu_utilization_mean_percent: 35,
|
|
gpu_utilization_maximum_percent: 85,
|
|
gpu_memory_maximum_mib: 9584,
|
|
process_peak_rss_mib: 2327,
|
|
queue_high_watermarks: { detector: 2, geometry: 2, temporal: 2, rolling: 2, threat: 2 },
|
|
queue_capacity: 2,
|
|
additional_inference_passes: 0,
|
|
repeat_count: 3,
|
|
all_repetitions_passed: true,
|
|
integrity_gate_passed: true,
|
|
operating_target_gate_passed: true,
|
|
thresholds: {
|
|
minimum_delivery_ratio: rate === 10 ? 0.999 : rate === 12 ? 0.995 : 0.95,
|
|
minimum_effective_world_state_fps: rate * 0.95,
|
|
maximum_world_state_completion_p95_ms: rate === 10 ? 125 : rate === 12 ? 150 : 175,
|
|
},
|
|
repetitions: [1, 2, 3].map((repetition) => ({
|
|
run_id: `m48s-load-prefetch-v1-${id}-a${repetition}`,
|
|
repetition,
|
|
delivered_world_states: delivered,
|
|
superseded_frames: superseded,
|
|
delivery_ratio: delivered / 4489,
|
|
effective_world_state_fps: rate * 0.975,
|
|
world_state_completion_age_p95_ms: 75,
|
|
hot_loop_decode_p95_ms: 14,
|
|
hot_loop_decode_maximum_ms: 42,
|
|
preadmission_decode_maximum_ms: 3633,
|
|
source_pacing_lateness_maximum_ms: 11,
|
|
operating_target_gate_passed: true,
|
|
result_sha256: "7".repeat(64),
|
|
frame_evidence_sha256: "8".repeat(64),
|
|
})),
|
|
});
|
|
return {
|
|
schema_version: "missioncore.m48s-fixed-class-detector-result-view/v1",
|
|
result_id: resultId,
|
|
created_at_utc: "2026-08-25T11:06:28Z",
|
|
status: "complete-reference-graph-shadow-passed-production-not-authorized",
|
|
bounded_question_accepted: true,
|
|
ground_truth: false,
|
|
source: {
|
|
source_session_id: "RAVNOVES00",
|
|
camera_source_id: "sensor.camera.right",
|
|
camera_raster: [800, 600],
|
|
evidence_frame_count: 1,
|
|
},
|
|
configuration: {
|
|
comparison_threshold: 0.5,
|
|
display_modes: ["source", "yolox", "dfine", "rf-detr"],
|
|
single_inference_per_frame: true,
|
|
geometry_owns_static_occupancy: true,
|
|
unknown_stationary_response: "route-around",
|
|
unknown_moving_response: "conservative-risk",
|
|
},
|
|
method: {
|
|
schema_version: "missioncore.laboratory-method/v1",
|
|
completeness: "complete",
|
|
execution_class: "ai-inference",
|
|
pipeline_id: "raw-kb4-fixed-class-risk-detector-tournament/v1",
|
|
components: [{
|
|
kind: "model",
|
|
name: "RF-DETR-L COCO",
|
|
version: "trt11-fp16",
|
|
role: "selected fixed-class risk detector",
|
|
identity_sha256: "9".repeat(64),
|
|
}],
|
|
},
|
|
metrics: {
|
|
candidates: [
|
|
candidate("yolox"),
|
|
candidate("dfine"),
|
|
candidate("rf-detr", {
|
|
label: "RF-DETR-L",
|
|
capacity_fps: 42.496232,
|
|
frame_253_dog_detected: true,
|
|
frame_253_dog_score: 0.741674,
|
|
selected: true,
|
|
}),
|
|
],
|
|
detector_load: {
|
|
duration_seconds: 1800.019643,
|
|
source_frames_consumed: 18008,
|
|
source_frame_replacements: 0,
|
|
effective_consumed_fps: 10.004444,
|
|
end_to_end_p95_ms: 32.41534,
|
|
completion_age_p95_ms: 40.620542,
|
|
gpu_utilization_mean_percent: 51.407556,
|
|
gpu_utilization_maximum_percent: 65,
|
|
gpu_memory_maximum_mib: 9556,
|
|
queue_maximum_depth: 1,
|
|
queue_capacity: 2,
|
|
failures: 0,
|
|
},
|
|
integrated_world_state: {
|
|
duration_seconds: 458.900859,
|
|
source_frames_admitted: 4489,
|
|
delivered_world_states: 4481,
|
|
superseded_frames: 8,
|
|
effective_world_state_fps: 9.764636,
|
|
world_state_completion_age_p95_ms: 74.733648,
|
|
world_state_completion_age_p99_ms: 102.62048,
|
|
world_state_completion_age_maximum_ms: 669.142137,
|
|
local_obstacle_map_output_age_p95_ms: 70.635141,
|
|
queue_high_watermarks: { detector: 2, geometry: 2, temporal: 2, rolling: 2, threat: 2 },
|
|
queue_capacity: 2,
|
|
gpu_utilization_mean_percent: 50.903371,
|
|
gpu_utilization_maximum_percent: 60,
|
|
gpu_memory_maximum_mib: 9576,
|
|
gpu_power_maximum_w: 153.51,
|
|
gpu_temperature_maximum_c: 39,
|
|
unique_component_count: 44979,
|
|
multi_frame_component_count: 15887,
|
|
maximum_component_publications: 219,
|
|
duplicate_component_ids_within_frame: 0,
|
|
advisory_family_counts: {
|
|
animal: 63,
|
|
"generic-obstacle": 123919,
|
|
"light-road-user": 329,
|
|
person: 5600,
|
|
vehicle: 55818,
|
|
},
|
|
semantic_hint_counts: { dog: 60, person: 5600, "geometry-only": 123919 },
|
|
motion_counts: { moving: 12252, stationary: 1433, unknown: 172044 },
|
|
additional_inference_passes: 0,
|
|
failures: 0,
|
|
},
|
|
runtime_hardening: {
|
|
schema_version: "missioncore.m48s-runtime-hardening-comparison/v1",
|
|
baseline: {
|
|
source_frames_admitted: 4489,
|
|
delivered_world_states: 4480,
|
|
superseded_frames: 9,
|
|
effective_world_state_fps: 9.750635,
|
|
world_state_completion_age_p95_ms: 76.886564,
|
|
world_state_completion_age_p99_ms: 100.661219,
|
|
world_state_completion_age_maximum_ms: 794.748644,
|
|
rolling_maximum_ms: 762.638263,
|
|
geometry_maximum_ms: 350.066302,
|
|
additional_inference_passes: 0,
|
|
},
|
|
hardened: {
|
|
source_frames_admitted: 4489,
|
|
delivered_world_states: 4488,
|
|
superseded_frames: 1,
|
|
effective_world_state_fps: 9.821942,
|
|
world_state_completion_age_p95_ms: 75.270655,
|
|
world_state_completion_age_p99_ms: 94.309605,
|
|
world_state_completion_age_maximum_ms: 449.893287,
|
|
rolling_maximum_ms: 28.044958,
|
|
geometry_maximum_ms: 49.174149,
|
|
additional_inference_passes: 0,
|
|
},
|
|
startup: {
|
|
baseline: { detector_ms: 397.356888, world_state_ms: 449.893287 },
|
|
prewarmed: { detector_ms: 27.662887, world_state_ms: 60.516957 },
|
|
prewarm_duration_ms: 420.721918,
|
|
prewarm_inference_passes: 1,
|
|
validation_frames: 1000,
|
|
},
|
|
},
|
|
load_envelope: {
|
|
schema_version: "missioncore.m48s-load-envelope-comparison/v2",
|
|
repeat_count_per_rate: 3,
|
|
production_rate_repeatability_passed: true,
|
|
reserve_12_fps_passed: true,
|
|
limit_15_fps_passed: true,
|
|
load_envelope_accepted: true,
|
|
compute_capacity_at_least_fps: 15,
|
|
bottleneck_interpretation: "cold-video-decode-isolated-before-admission-no-steady-gpu-saturation",
|
|
scenarios: [
|
|
loadScenario("production-10fps", "production-rate", 10, 4489, 0),
|
|
loadScenario("reserve-12fps", "reserve-gate", 12, 4488, 1),
|
|
loadScenario("limit-15fps", "limit-discovery", 15, 4488, 1),
|
|
],
|
|
},
|
|
},
|
|
decision: {
|
|
selected_candidate: "rf-detr",
|
|
ready_for_reference_graph_shadow: true,
|
|
integrated_world_state_gate_evaluated: true,
|
|
integrated_world_state_gate_passed: true,
|
|
detector_replacement_authorized: false,
|
|
load_envelope_evaluated: true,
|
|
production_rate_repeatability_passed: true,
|
|
reserve_12_fps_passed: true,
|
|
limit_15_fps_passed: true,
|
|
load_envelope_accepted: true,
|
|
production_accepted: false,
|
|
},
|
|
limitations: ["No independent semantic ground truth."],
|
|
authority,
|
|
frames: [{
|
|
frame_id: "000253",
|
|
source_sequence: 253,
|
|
counts: { yolox: 4, dfine: 7, "rf-detr": 8 },
|
|
}],
|
|
access: "read-only",
|
|
};
|
|
}
|
|
|
|
test("M4.8S result exposes complete graph load without production authority", async () => {
|
|
const result = await fetchM48SFixedClassDetectorResult(resultId, {
|
|
fetcher: async (url) => {
|
|
assert.equal(
|
|
url,
|
|
`/api/v1/laboratory/m48s/fixed-class-detector/${resultId}`,
|
|
);
|
|
return response(resultPayload());
|
|
},
|
|
});
|
|
|
|
assert.equal(result.metrics.detectorLoad.sourceFramesConsumed, 18008);
|
|
assert.equal(result.metrics.detectorLoad.completionAgeP95Ms, 40.620542);
|
|
assert.equal(result.metrics.candidates[2].frame253DogScore, 0.741674);
|
|
assert.equal(result.metrics.integratedWorldState.deliveredWorldStates, 4481);
|
|
assert.equal(result.metrics.integratedWorldState.worldStateCompletionAgeP95Ms, 74.733648);
|
|
assert.equal(result.metrics.integratedWorldState.additionalInferencePasses, 0);
|
|
assert.equal(result.metrics.runtimeHardening.baseline.supersededFrames, 9);
|
|
assert.equal(result.metrics.runtimeHardening.hardened.supersededFrames, 1);
|
|
assert.equal(result.metrics.runtimeHardening.startup.prewarmed.worldStateMs, 60.516957);
|
|
assert.equal(result.metrics.loadEnvelope.scenarios[0].operatingTargetGatePassed, true);
|
|
assert.equal(result.metrics.loadEnvelope.scenarios[1].deliveredWorldStates, 4488);
|
|
assert.equal(result.metrics.loadEnvelope.scenarios[2].repetitions.length, 3);
|
|
assert.equal(result.metrics.loadEnvelope.scenarios[0].hotLoopDecodeP95Ms, 14);
|
|
assert.equal(result.metrics.loadEnvelope.computeCapacityAtLeastFps, 15);
|
|
assert.equal(result.decision.loadEnvelopeAccepted, true);
|
|
assert.equal(result.decision.integratedWorldStateGatePassed, true);
|
|
assert.equal(result.decision.productionAccepted, false);
|
|
assert.equal(result.authority.navigationOrSafetyAccepted, false);
|
|
});
|
|
|
|
test("M4.8S keeps the immutable v1 load-envelope result readable", async () => {
|
|
const payload = resultPayload();
|
|
payload.metrics.load_envelope = {
|
|
schema_version: "missioncore.m48s-load-envelope-comparison/v1",
|
|
production_rate_repeatability_passed: false,
|
|
reserve_12_fps_passed: true,
|
|
limit_15_fps_passed: true,
|
|
load_envelope_accepted: false,
|
|
compute_capacity_at_least_fps: 15,
|
|
bottleneck_interpretation: "rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation",
|
|
scenarios: payload.metrics.load_envelope.scenarios.map((scenario, index) => ({
|
|
id: scenario.id,
|
|
load_purpose: scenario.load_purpose,
|
|
requested_source_rate_hz: scenario.requested_source_rate_hz,
|
|
source_frames_admitted: scenario.source_frames_admitted,
|
|
delivered_world_states: index === 0 ? 4483 : scenario.delivered_world_states,
|
|
superseded_frames: index === 0 ? 6 : scenario.superseded_frames,
|
|
delivery_ratio: index === 0 ? 4483 / 4489 : scenario.delivery_ratio,
|
|
effective_world_state_fps: scenario.effective_world_state_fps,
|
|
world_state_completion_age_p95_ms: scenario.world_state_completion_age_p95_ms,
|
|
world_state_completion_age_p99_ms: scenario.world_state_completion_age_p99_ms,
|
|
world_state_completion_age_maximum_ms: scenario.world_state_completion_age_maximum_ms,
|
|
decode_p95_ms: scenario.hot_loop_decode_p95_ms,
|
|
decode_maximum_ms: 3200,
|
|
detector_p95_ms: scenario.detector_p95_ms,
|
|
detector_maximum_ms: scenario.detector_maximum_ms,
|
|
gpu_utilization_mean_percent: scenario.gpu_utilization_mean_percent,
|
|
gpu_utilization_maximum_percent: scenario.gpu_utilization_maximum_percent,
|
|
gpu_memory_maximum_mib: scenario.gpu_memory_maximum_mib,
|
|
process_peak_rss_mib: scenario.process_peak_rss_mib,
|
|
queue_high_watermarks: scenario.queue_high_watermarks,
|
|
queue_capacity: 2,
|
|
additional_inference_passes: 0,
|
|
integrity_gate_passed: true,
|
|
operating_target_gate_passed: index !== 0,
|
|
thresholds: scenario.thresholds,
|
|
frame_evidence_sha256: "8".repeat(64),
|
|
})),
|
|
};
|
|
payload.decision.production_rate_repeatability_passed = false;
|
|
payload.decision.load_envelope_accepted = false;
|
|
|
|
const result = await fetchM48SFixedClassDetectorResult(resultId, {
|
|
fetcher: async () => response(payload),
|
|
});
|
|
|
|
assert.equal(result.metrics.loadEnvelope.repeatCountPerRate, 1);
|
|
assert.equal(result.metrics.loadEnvelope.loadEnvelopeAccepted, false);
|
|
assert.equal(result.metrics.loadEnvelope.scenarios[0].operatingTargetGatePassed, false);
|
|
});
|
|
|
|
test("M4.8S frame binds exact camera endpoint and risk-only boxes", async () => {
|
|
const frame = await fetchM48SFixedClassDetectorFrame(resultId, "000253", {
|
|
fetcher: async () => response({
|
|
schema_version: "missioncore.m48s-fixed-class-detector-frame/v1",
|
|
result_id: resultId,
|
|
frame_id: "000253",
|
|
source_sequence: 253,
|
|
camera: {
|
|
media_type: "image/jpeg",
|
|
width: 800,
|
|
height: 600,
|
|
exact_source_frame: true,
|
|
},
|
|
comparison_threshold: 0.5,
|
|
detections: {
|
|
yolox: [],
|
|
dfine: [{ label: "skateboard", score: 0.782227, bbox_xyxy: [236, 334, 274, 372] }],
|
|
"rf-detr": [{ label: "dog", score: 0.741674, bbox_xyxy: [235, 336, 273, 376] }],
|
|
},
|
|
ground_truth_available: false,
|
|
authority,
|
|
access: "read-only",
|
|
}),
|
|
});
|
|
|
|
assert.equal(frame.detections["rf-detr"][0].label, "dog");
|
|
assert.equal(
|
|
frame.cameraUrl,
|
|
`/api/v1/laboratory/m48s/fixed-class-detector/${resultId}/frames/000253/camera`,
|
|
);
|
|
assert.equal(frame.groundTruthAvailable, false);
|
|
});
|
|
|
|
test("M4.8S adapter rejects any navigation authority escalation", async () => {
|
|
await assert.rejects(
|
|
fetchM48SFixedClassDetectorResult(resultId, {
|
|
fetcher: async () => response({
|
|
...resultPayload(),
|
|
authority: { ...authority, navigation_or_safety_accepted: true },
|
|
}),
|
|
}),
|
|
/navigation_or_safety_accepted/,
|
|
);
|
|
});
|