6 Commits
26 changed files with 3551 additions and 82 deletions
@@ -2,6 +2,7 @@ import type { LaboratoryFetch } from "./advancedResults";
const RESULT_ID = /^m48s-fixed-class-detector-lab-[a-f0-9]{64}$/; const RESULT_ID = /^m48s-fixed-class-detector-lab-[a-f0-9]{64}$/;
const FRAME_ID = /^[0-9]{6}$/; const FRAME_ID = /^[0-9]{6}$/;
const SHA256 = /^[a-f0-9]{64}$/;
const MODES = ["source", "yolox", "dfine", "rf-detr"] as const; const MODES = ["source", "yolox", "dfine", "rf-detr"] as const;
const DETECTOR_MODES = ["yolox", "dfine", "rf-detr"] as const; const DETECTOR_MODES = ["yolox", "dfine", "rf-detr"] as const;
const RESULT_STATUSES = [ const RESULT_STATUSES = [
@@ -80,6 +81,100 @@ export interface M48SIntegratedWorldState {
failures: number; failures: number;
} }
export interface M48SRuntimeHardeningFullRun {
sourceFramesAdmitted: number;
deliveredWorldStates: number;
supersededFrames: number;
effectiveWorldStateFps: number;
worldStateCompletionAgeP95Ms: number;
worldStateCompletionAgeP99Ms: number;
worldStateCompletionAgeMaximumMs: number;
rollingMaximumMs: number;
geometryMaximumMs: number;
additionalInferencePasses: number;
}
export interface M48SRuntimeHardeningComparison {
baseline: M48SRuntimeHardeningFullRun;
hardened: M48SRuntimeHardeningFullRun;
startup: {
baseline: { detectorMs: number; worldStateMs: number };
prewarmed: { detectorMs: number; worldStateMs: number };
prewarmDurationMs: number;
prewarmInferencePasses: number;
validationFrames: number;
};
}
export interface M48SLoadEnvelopeScenario {
id: "production-10fps" | "reserve-12fps" | "limit-15fps";
loadPurpose: "production-rate" | "reserve-gate" | "limit-discovery";
requestedSourceRateHz: number;
sourceFramesAdmitted: number;
deliveredWorldStates: number;
supersededFrames: number;
deliveryRatio: number;
effectiveWorldStateFps: number;
worldStateCompletionAgeP95Ms: number;
worldStateCompletionAgeP99Ms: number;
worldStateCompletionAgeMaximumMs: number;
hotLoopDecodeP95Ms: number;
hotLoopDecodeMaximumMs: number;
preadmissionDecodeMaximumMs: number;
sourcePrefetchPreparationMaximumMs: number;
sourcePacingLatenessP95Ms: number;
sourcePacingLatenessMaximumMs: number;
detectorP95Ms: number;
detectorMaximumMs: number;
gpuUtilizationMeanPercent: number;
gpuUtilizationMaximumPercent: number;
gpuMemoryMaximumMib: number;
processPeakRssMib: number;
queueHighWatermarks: Readonly<Record<"detector" | "geometry" | "temporal" | "rolling" | "threat", number>>;
queueCapacity: number;
additionalInferencePasses: number;
repeatCount: number;
allRepetitionsPassed: boolean;
integrityGatePassed: boolean;
operatingTargetGatePassed: boolean;
thresholds: {
minimumDeliveryRatio: number;
minimumEffectiveWorldStateFps: number;
maximumWorldStateCompletionP95Ms: number;
};
repetitions: readonly M48SLoadEnvelopeRepetition[];
}
export interface M48SLoadEnvelopeRepetition {
runId: string;
repetition: 1 | 2 | 3;
deliveredWorldStates: number;
supersededFrames: number;
deliveryRatio: number;
effectiveWorldStateFps: number;
worldStateCompletionAgeP95Ms: number;
hotLoopDecodeP95Ms: number;
hotLoopDecodeMaximumMs: number;
preadmissionDecodeMaximumMs: number;
sourcePacingLatenessMaximumMs: number;
operatingTargetGatePassed: boolean;
resultSha256: string;
frameEvidenceSha256: string;
}
export interface M48SLoadEnvelopeComparison {
repeatCountPerRate: number;
productionRateRepeatabilityPassed: boolean;
reserve12FpsPassed: boolean;
limit15FpsPassed: boolean;
loadEnvelopeAccepted: boolean;
computeCapacityAtLeastFps: 15;
bottleneckInterpretation:
| "rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation"
| "cold-video-decode-isolated-before-admission-no-steady-gpu-saturation";
scenarios: readonly M48SLoadEnvelopeScenario[];
}
export interface M48SFixedClassDetectorResult { export interface M48SFixedClassDetectorResult {
resultId: string; resultId: string;
createdAtUtc: string; createdAtUtc: string;
@@ -118,6 +213,8 @@ export interface M48SFixedClassDetectorResult {
failures: number; failures: number;
}; };
integratedWorldState: M48SIntegratedWorldState | null; integratedWorldState: M48SIntegratedWorldState | null;
runtimeHardening: M48SRuntimeHardeningComparison | null;
loadEnvelope: M48SLoadEnvelopeComparison | null;
}; };
decision: { decision: {
selectedCandidate: "rf-detr"; selectedCandidate: "rf-detr";
@@ -125,6 +222,11 @@ export interface M48SFixedClassDetectorResult {
integratedWorldStateGateEvaluated: boolean; integratedWorldStateGateEvaluated: boolean;
integratedWorldStateGatePassed: boolean; integratedWorldStateGatePassed: boolean;
detectorReplacementAuthorized: false; detectorReplacementAuthorized: false;
loadEnvelopeEvaluated: boolean;
productionRateRepeatabilityPassed: boolean;
reserve12FpsPassed: boolean;
limit15FpsPassed: boolean;
loadEnvelopeAccepted: boolean;
productionAccepted: false; productionAccepted: false;
}; };
limitations: readonly string[]; limitations: readonly string[];
@@ -366,6 +468,288 @@ function integratedWorldStateValue(value: unknown): M48SIntegratedWorldState {
}; };
} }
function runtimeHardeningFullRunValue(
value: unknown,
label: string,
): M48SRuntimeHardeningFullRun {
const run = objectValue(value, label);
return {
sourceFramesAdmitted: integerValue(run.source_frames_admitted, `${label}.source_frames_admitted`),
deliveredWorldStates: integerValue(run.delivered_world_states, `${label}.delivered_world_states`),
supersededFrames: integerValue(run.superseded_frames, `${label}.superseded_frames`),
effectiveWorldStateFps: numberValue(run.effective_world_state_fps, `${label}.effective_world_state_fps`),
worldStateCompletionAgeP95Ms: numberValue(run.world_state_completion_age_p95_ms, `${label}.world_state_completion_age_p95_ms`),
worldStateCompletionAgeP99Ms: numberValue(run.world_state_completion_age_p99_ms, `${label}.world_state_completion_age_p99_ms`),
worldStateCompletionAgeMaximumMs: numberValue(run.world_state_completion_age_maximum_ms, `${label}.world_state_completion_age_maximum_ms`),
rollingMaximumMs: numberValue(run.rolling_maximum_ms, `${label}.rolling_maximum_ms`),
geometryMaximumMs: numberValue(run.geometry_maximum_ms, `${label}.geometry_maximum_ms`),
additionalInferencePasses: integerValue(run.additional_inference_passes, `${label}.additional_inference_passes`),
};
}
function runtimeHardeningComparisonValue(value: unknown): M48SRuntimeHardeningComparison {
const comparison = objectValue(value, "M4.8S.metrics.runtime_hardening");
exact(
comparison.schema_version,
"missioncore.m48s-runtime-hardening-comparison/v1",
"M4.8S.metrics.runtime_hardening.schema_version",
);
const startup = objectValue(comparison.startup, "M4.8S.runtime_hardening.startup");
const startupFrame = (
candidate: unknown,
label: string,
): { detectorMs: number; worldStateMs: number } => {
const frame = objectValue(candidate, label);
return {
detectorMs: numberValue(frame.detector_ms, `${label}.detector_ms`),
worldStateMs: numberValue(frame.world_state_ms, `${label}.world_state_ms`),
};
};
return {
baseline: runtimeHardeningFullRunValue(
comparison.baseline,
"M4.8S.runtime_hardening.baseline",
),
hardened: runtimeHardeningFullRunValue(
comparison.hardened,
"M4.8S.runtime_hardening.hardened",
),
startup: {
baseline: startupFrame(startup.baseline, "M4.8S.runtime_hardening.startup.baseline"),
prewarmed: startupFrame(startup.prewarmed, "M4.8S.runtime_hardening.startup.prewarmed"),
prewarmDurationMs: numberValue(
startup.prewarm_duration_ms,
"M4.8S.runtime_hardening.startup.prewarm_duration_ms",
),
prewarmInferencePasses: integerValue(
startup.prewarm_inference_passes,
"M4.8S.runtime_hardening.startup.prewarm_inference_passes",
),
validationFrames: integerValue(
startup.validation_frames,
"M4.8S.runtime_hardening.startup.validation_frames",
),
},
};
}
function loadEnvelopeComparisonValue(value: unknown): M48SLoadEnvelopeComparison {
const comparison = objectValue(value, "M4.8S.metrics.load_envelope");
if (comparison.schema_version === "missioncore.m48s-load-envelope-comparison/v1") {
exact(comparison.production_rate_repeatability_passed, false, "M4.8S.load_envelope.production_rate_repeatability_passed");
exact(comparison.reserve_12_fps_passed, true, "M4.8S.load_envelope.reserve_12_fps_passed");
exact(comparison.limit_15_fps_passed, true, "M4.8S.load_envelope.limit_15_fps_passed");
exact(comparison.load_envelope_accepted, false, "M4.8S.load_envelope.load_envelope_accepted");
exact(comparison.compute_capacity_at_least_fps, 15, "M4.8S.load_envelope.compute_capacity_at_least_fps");
exact(
comparison.bottleneck_interpretation,
"rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation",
"M4.8S.load_envelope.bottleneck_interpretation",
);
const ids = ["production-10fps", "reserve-12fps", "limit-15fps"] as const;
const purposes = ["production-rate", "reserve-gate", "limit-discovery"] as const;
const targetGates = [false, true, true] as const;
const scenarios = arrayValue(comparison.scenarios, "M4.8S.load_envelope.scenarios").map((value, index) => {
const label = `M4.8S.load_envelope.scenarios[${index}]`;
const scenario = objectValue(value, label);
const id = ids[index];
const purpose = purposes[index];
const targetGate = targetGates[index];
if (id === undefined || purpose === undefined || targetGate === undefined) {
throw new M48SFixedClassDetectorContractError("M4.8S.load_envelope.scenarios: лишний legacy-сценарий.");
}
exact(scenario.id, id, `${label}.id`);
exact(scenario.load_purpose, purpose, `${label}.load_purpose`);
exact(scenario.integrity_gate_passed, true, `${label}.integrity_gate_passed`);
exact(scenario.operating_target_gate_passed, targetGate, `${label}.operating_target_gate_passed`);
exact(scenario.additional_inference_passes, 0, `${label}.additional_inference_passes`);
const queues = integerRecordValue(scenario.queue_high_watermarks, `${label}.queue_high_watermarks`);
const thresholds = objectValue(scenario.thresholds, `${label}.thresholds`);
const frameEvidenceSha256 = textValue(scenario.frame_evidence_sha256, `${label}.frame_evidence_sha256`);
if (!SHA256.test(frameEvidenceSha256)) {
throw new M48SFixedClassDetectorContractError(`${label}.frame_evidence_sha256: нарушен SHA-256.`);
}
const legacyDecodeP95 = numberValue(scenario.decode_p95_ms, `${label}.decode_p95_ms`);
const legacyDecodeMaximum = numberValue(scenario.decode_maximum_ms, `${label}.decode_maximum_ms`);
return {
id,
loadPurpose: purpose,
requestedSourceRateHz: numberValue(scenario.requested_source_rate_hz, `${label}.requested_source_rate_hz`),
sourceFramesAdmitted: integerValue(scenario.source_frames_admitted, `${label}.source_frames_admitted`),
deliveredWorldStates: integerValue(scenario.delivered_world_states, `${label}.delivered_world_states`),
supersededFrames: integerValue(scenario.superseded_frames, `${label}.superseded_frames`),
deliveryRatio: numberValue(scenario.delivery_ratio, `${label}.delivery_ratio`),
effectiveWorldStateFps: numberValue(scenario.effective_world_state_fps, `${label}.effective_world_state_fps`),
worldStateCompletionAgeP95Ms: numberValue(scenario.world_state_completion_age_p95_ms, `${label}.world_state_completion_age_p95_ms`),
worldStateCompletionAgeP99Ms: numberValue(scenario.world_state_completion_age_p99_ms, `${label}.world_state_completion_age_p99_ms`),
worldStateCompletionAgeMaximumMs: numberValue(scenario.world_state_completion_age_maximum_ms, `${label}.world_state_completion_age_maximum_ms`),
hotLoopDecodeP95Ms: legacyDecodeP95,
hotLoopDecodeMaximumMs: legacyDecodeMaximum,
preadmissionDecodeMaximumMs: legacyDecodeMaximum,
sourcePrefetchPreparationMaximumMs: 0,
sourcePacingLatenessP95Ms: 0,
sourcePacingLatenessMaximumMs: 0,
detectorP95Ms: numberValue(scenario.detector_p95_ms, `${label}.detector_p95_ms`),
detectorMaximumMs: numberValue(scenario.detector_maximum_ms, `${label}.detector_maximum_ms`),
gpuUtilizationMeanPercent: numberValue(scenario.gpu_utilization_mean_percent, `${label}.gpu_utilization_mean_percent`),
gpuUtilizationMaximumPercent: numberValue(scenario.gpu_utilization_maximum_percent, `${label}.gpu_utilization_maximum_percent`),
gpuMemoryMaximumMib: numberValue(scenario.gpu_memory_maximum_mib, `${label}.gpu_memory_maximum_mib`),
processPeakRssMib: numberValue(scenario.process_peak_rss_mib, `${label}.process_peak_rss_mib`),
queueHighWatermarks: queues as M48SLoadEnvelopeScenario["queueHighWatermarks"],
queueCapacity: integerValue(scenario.queue_capacity, `${label}.queue_capacity`),
additionalInferencePasses: 0,
repeatCount: 1,
allRepetitionsPassed: targetGate,
integrityGatePassed: true,
operatingTargetGatePassed: targetGate,
thresholds: {
minimumDeliveryRatio: numberValue(thresholds.minimum_delivery_ratio, `${label}.thresholds.minimum_delivery_ratio`),
minimumEffectiveWorldStateFps: numberValue(thresholds.minimum_effective_world_state_fps, `${label}.thresholds.minimum_effective_world_state_fps`),
maximumWorldStateCompletionP95Ms: numberValue(thresholds.maximum_world_state_completion_p95_ms, `${label}.thresholds.maximum_world_state_completion_p95_ms`),
},
repetitions: [],
};
});
if (scenarios.length !== 3) {
throw new M48SFixedClassDetectorContractError("M4.8S.load_envelope.scenarios: неполный legacy-набор.");
}
return {
repeatCountPerRate: 1,
productionRateRepeatabilityPassed: false,
reserve12FpsPassed: true,
limit15FpsPassed: true,
loadEnvelopeAccepted: false,
computeCapacityAtLeastFps: 15,
bottleneckInterpretation: "rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation",
scenarios,
};
}
exact(
comparison.schema_version,
"missioncore.m48s-load-envelope-comparison/v2",
"M4.8S.metrics.load_envelope.schema_version",
);
exact(comparison.repeat_count_per_rate, 3, "M4.8S.load_envelope.repeat_count_per_rate");
exact(comparison.production_rate_repeatability_passed, true, "M4.8S.load_envelope.production_rate_repeatability_passed");
exact(comparison.reserve_12_fps_passed, true, "M4.8S.load_envelope.reserve_12_fps_passed");
exact(comparison.limit_15_fps_passed, true, "M4.8S.load_envelope.limit_15_fps_passed");
exact(comparison.load_envelope_accepted, true, "M4.8S.load_envelope.load_envelope_accepted");
exact(comparison.compute_capacity_at_least_fps, 15, "M4.8S.load_envelope.compute_capacity_at_least_fps");
exact(
comparison.bottleneck_interpretation,
"cold-video-decode-isolated-before-admission-no-steady-gpu-saturation",
"M4.8S.load_envelope.bottleneck_interpretation",
);
const ids = ["production-10fps", "reserve-12fps", "limit-15fps"] as const;
const purposes = ["production-rate", "reserve-gate", "limit-discovery"] as const;
const scenarios = arrayValue(comparison.scenarios, "M4.8S.load_envelope.scenarios").map((value, index) => {
const label = `M4.8S.load_envelope.scenarios[${index}]`;
const scenario = objectValue(value, label);
const id = ids[index];
const purpose = purposes[index];
if (id === undefined || purpose === undefined) {
throw new M48SFixedClassDetectorContractError("M4.8S.load_envelope.scenarios: лишний сценарий.");
}
exact(scenario.id, id, `${label}.id`);
exact(scenario.load_purpose, purpose, `${label}.load_purpose`);
exact(scenario.repeat_count, 3, `${label}.repeat_count`);
exact(scenario.all_repetitions_passed, true, `${label}.all_repetitions_passed`);
exact(scenario.integrity_gate_passed, true, `${label}.integrity_gate_passed`);
exact(scenario.operating_target_gate_passed, true, `${label}.operating_target_gate_passed`);
exact(scenario.additional_inference_passes, 0, `${label}.additional_inference_passes`);
const queues = integerRecordValue(scenario.queue_high_watermarks, `${label}.queue_high_watermarks`);
for (const key of ["detector", "geometry", "temporal", "rolling", "threat"] as const) {
if (!(key in queues)) {
throw new M48SFixedClassDetectorContractError(`${label}.queue_high_watermarks.${key}: отсутствует.`);
}
}
const thresholds = objectValue(scenario.thresholds, `${label}.thresholds`);
const repetitions = arrayValue(scenario.repetitions, `${label}.repetitions`).map((value, repetitionIndex) => {
const repetitionLabel = `${label}.repetitions[${repetitionIndex}]`;
const repetition = objectValue(value, repetitionLabel);
const repetitionNumber = repetitionIndex + 1;
exact(repetition.repetition, repetitionNumber, `${repetitionLabel}.repetition`);
exact(repetition.operating_target_gate_passed, true, `${repetitionLabel}.operating_target_gate_passed`);
const runId = textValue(repetition.run_id, `${repetitionLabel}.run_id`);
const resultSha256 = textValue(repetition.result_sha256, `${repetitionLabel}.result_sha256`);
const frameEvidenceSha256 = textValue(repetition.frame_evidence_sha256, `${repetitionLabel}.frame_evidence_sha256`);
if (!SHA256.test(resultSha256) || !SHA256.test(frameEvidenceSha256)) {
throw new M48SFixedClassDetectorContractError(`${repetitionLabel}: нарушен SHA-256.`);
}
return {
runId,
repetition: repetitionNumber as 1 | 2 | 3,
deliveredWorldStates: integerValue(repetition.delivered_world_states, `${repetitionLabel}.delivered_world_states`),
supersededFrames: integerValue(repetition.superseded_frames, `${repetitionLabel}.superseded_frames`),
deliveryRatio: numberValue(repetition.delivery_ratio, `${repetitionLabel}.delivery_ratio`),
effectiveWorldStateFps: numberValue(repetition.effective_world_state_fps, `${repetitionLabel}.effective_world_state_fps`),
worldStateCompletionAgeP95Ms: numberValue(repetition.world_state_completion_age_p95_ms, `${repetitionLabel}.world_state_completion_age_p95_ms`),
hotLoopDecodeP95Ms: numberValue(repetition.hot_loop_decode_p95_ms, `${repetitionLabel}.hot_loop_decode_p95_ms`),
hotLoopDecodeMaximumMs: numberValue(repetition.hot_loop_decode_maximum_ms, `${repetitionLabel}.hot_loop_decode_maximum_ms`),
preadmissionDecodeMaximumMs: numberValue(repetition.preadmission_decode_maximum_ms, `${repetitionLabel}.preadmission_decode_maximum_ms`),
sourcePacingLatenessMaximumMs: numberValue(repetition.source_pacing_lateness_maximum_ms, `${repetitionLabel}.source_pacing_lateness_maximum_ms`),
operatingTargetGatePassed: true as const,
resultSha256,
frameEvidenceSha256,
};
});
if (repetitions.length !== 3) {
throw new M48SFixedClassDetectorContractError(`${label}.repetitions: требуется три повтора.`);
}
return {
id,
loadPurpose: purpose,
requestedSourceRateHz: numberValue(scenario.requested_source_rate_hz, `${label}.requested_source_rate_hz`),
sourceFramesAdmitted: integerValue(scenario.source_frames_admitted, `${label}.source_frames_admitted`),
deliveredWorldStates: integerValue(scenario.delivered_world_states, `${label}.delivered_world_states`),
supersededFrames: integerValue(scenario.superseded_frames, `${label}.superseded_frames`),
deliveryRatio: numberValue(scenario.delivery_ratio, `${label}.delivery_ratio`),
effectiveWorldStateFps: numberValue(scenario.effective_world_state_fps, `${label}.effective_world_state_fps`),
worldStateCompletionAgeP95Ms: numberValue(scenario.world_state_completion_age_p95_ms, `${label}.world_state_completion_age_p95_ms`),
worldStateCompletionAgeP99Ms: numberValue(scenario.world_state_completion_age_p99_ms, `${label}.world_state_completion_age_p99_ms`),
worldStateCompletionAgeMaximumMs: numberValue(scenario.world_state_completion_age_maximum_ms, `${label}.world_state_completion_age_maximum_ms`),
hotLoopDecodeP95Ms: numberValue(scenario.hot_loop_decode_p95_ms, `${label}.hot_loop_decode_p95_ms`),
hotLoopDecodeMaximumMs: numberValue(scenario.hot_loop_decode_maximum_ms, `${label}.hot_loop_decode_maximum_ms`),
preadmissionDecodeMaximumMs: numberValue(scenario.preadmission_decode_maximum_ms, `${label}.preadmission_decode_maximum_ms`),
sourcePrefetchPreparationMaximumMs: numberValue(scenario.source_prefetch_preparation_maximum_ms, `${label}.source_prefetch_preparation_maximum_ms`),
sourcePacingLatenessP95Ms: numberValue(scenario.source_pacing_lateness_p95_ms, `${label}.source_pacing_lateness_p95_ms`),
sourcePacingLatenessMaximumMs: numberValue(scenario.source_pacing_lateness_maximum_ms, `${label}.source_pacing_lateness_maximum_ms`),
detectorP95Ms: numberValue(scenario.detector_p95_ms, `${label}.detector_p95_ms`),
detectorMaximumMs: numberValue(scenario.detector_maximum_ms, `${label}.detector_maximum_ms`),
gpuUtilizationMeanPercent: numberValue(scenario.gpu_utilization_mean_percent, `${label}.gpu_utilization_mean_percent`),
gpuUtilizationMaximumPercent: numberValue(scenario.gpu_utilization_maximum_percent, `${label}.gpu_utilization_maximum_percent`),
gpuMemoryMaximumMib: numberValue(scenario.gpu_memory_maximum_mib, `${label}.gpu_memory_maximum_mib`),
processPeakRssMib: numberValue(scenario.process_peak_rss_mib, `${label}.process_peak_rss_mib`),
queueHighWatermarks: queues as M48SLoadEnvelopeScenario["queueHighWatermarks"],
queueCapacity: integerValue(scenario.queue_capacity, `${label}.queue_capacity`),
additionalInferencePasses: 0,
repeatCount: 3 as const,
allRepetitionsPassed: true as const,
integrityGatePassed: true as const,
operatingTargetGatePassed: true as const,
thresholds: {
minimumDeliveryRatio: numberValue(thresholds.minimum_delivery_ratio, `${label}.thresholds.minimum_delivery_ratio`),
minimumEffectiveWorldStateFps: numberValue(thresholds.minimum_effective_world_state_fps, `${label}.thresholds.minimum_effective_world_state_fps`),
maximumWorldStateCompletionP95Ms: numberValue(thresholds.maximum_world_state_completion_p95_ms, `${label}.thresholds.maximum_world_state_completion_p95_ms`),
},
repetitions,
};
});
if (scenarios.length !== ids.length) {
throw new M48SFixedClassDetectorContractError("M4.8S.load_envelope.scenarios: неполный набор.");
}
return {
repeatCountPerRate: 3,
productionRateRepeatabilityPassed: true,
reserve12FpsPassed: true,
limit15FpsPassed: true,
loadEnvelopeAccepted: true,
computeCapacityAtLeastFps: 15,
bottleneckInterpretation: "cold-video-decode-isolated-before-admission-no-steady-gpu-saturation",
scenarios,
};
}
function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorResult { function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorResult {
const payload = objectValue(value, "M4.8S"); const payload = objectValue(value, "M4.8S");
exact( exact(
@@ -384,6 +768,9 @@ function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorRe
const metrics = objectValue(payload.metrics, "M4.8S.metrics"); const metrics = objectValue(payload.metrics, "M4.8S.metrics");
const load = objectValue(metrics.detector_load, "M4.8S.metrics.detector_load"); const load = objectValue(metrics.detector_load, "M4.8S.metrics.detector_load");
const decision = objectValue(payload.decision, "M4.8S.decision"); const decision = objectValue(payload.decision, "M4.8S.decision");
const loadEnvelope = metrics.load_envelope === undefined
? null
: loadEnvelopeComparisonValue(metrics.load_envelope);
const cameraRaster = arrayValue(source.camera_raster, "M4.8S.source.camera_raster"); const cameraRaster = arrayValue(source.camera_raster, "M4.8S.source.camera_raster");
if (cameraRaster.length !== 2) { if (cameraRaster.length !== 2) {
throw new M48SFixedClassDetectorContractError("M4.8S.source.camera_raster: нарушен размер."); throw new M48SFixedClassDetectorContractError("M4.8S.source.camera_raster: нарушен размер.");
@@ -412,6 +799,13 @@ function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorRe
exact(decision.detector_replacement_authorized, false, "M4.8S.decision.detector_replacement_authorized"); exact(decision.detector_replacement_authorized, false, "M4.8S.decision.detector_replacement_authorized");
} }
exact(decision.production_accepted, false, "M4.8S.decision.production_accepted"); exact(decision.production_accepted, false, "M4.8S.decision.production_accepted");
if (loadEnvelope !== null) {
exact(decision.load_envelope_evaluated, true, "M4.8S.decision.load_envelope_evaluated");
exact(decision.production_rate_repeatability_passed, loadEnvelope.productionRateRepeatabilityPassed, "M4.8S.decision.production_rate_repeatability_passed");
exact(decision.reserve_12_fps_passed, loadEnvelope.reserve12FpsPassed, "M4.8S.decision.reserve_12_fps_passed");
exact(decision.limit_15_fps_passed, loadEnvelope.limit15FpsPassed, "M4.8S.decision.limit_15_fps_passed");
exact(decision.load_envelope_accepted, loadEnvelope.loadEnvelopeAccepted, "M4.8S.decision.load_envelope_accepted");
}
const frames = arrayValue(payload.frames, "M4.8S.frames").map(descriptorValue); const frames = arrayValue(payload.frames, "M4.8S.frames").map(descriptorValue);
const evidenceFrameCount = integerValue(source.evidence_frame_count, "M4.8S.source.evidence_frame_count"); const evidenceFrameCount = integerValue(source.evidence_frame_count, "M4.8S.source.evidence_frame_count");
if (frames.length !== evidenceFrameCount || new Set(frames.map((frame) => frame.frameId)).size !== frames.length) { if (frames.length !== evidenceFrameCount || new Set(frames.map((frame) => frame.frameId)).size !== frames.length) {
@@ -457,6 +851,10 @@ function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorRe
integratedWorldState: integratedGate integratedWorldState: integratedGate
? integratedWorldStateValue(metrics.integrated_world_state) ? integratedWorldStateValue(metrics.integrated_world_state)
: null, : null,
runtimeHardening: metrics.runtime_hardening === undefined
? null
: runtimeHardeningComparisonValue(metrics.runtime_hardening),
loadEnvelope,
}, },
decision: { decision: {
selectedCandidate: "rf-detr", selectedCandidate: "rf-detr",
@@ -464,6 +862,11 @@ function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorRe
integratedWorldStateGateEvaluated: integratedGate, integratedWorldStateGateEvaluated: integratedGate,
integratedWorldStateGatePassed: integratedGate, integratedWorldStateGatePassed: integratedGate,
detectorReplacementAuthorized: false, detectorReplacementAuthorized: false,
loadEnvelopeEvaluated: loadEnvelope !== null,
productionRateRepeatabilityPassed: loadEnvelope?.productionRateRepeatabilityPassed ?? false,
reserve12FpsPassed: loadEnvelope?.reserve12FpsPassed ?? false,
limit15FpsPassed: loadEnvelope?.limit15FpsPassed ?? false,
loadEnvelopeAccepted: loadEnvelope?.loadEnvelopeAccepted ?? false,
productionAccepted: false, productionAccepted: false,
}, },
limitations: arrayValue(payload.limitations, "M4.8S.limitations").map((item, index) => textValue(item, `M4.8S.limitations[${index}]`)), limitations: arrayValue(payload.limitations, "M4.8S.limitations").map((item, index) => textValue(item, `M4.8S.limitations[${index}]`)),
@@ -19,9 +19,24 @@ export function M48SFixedClassDetectorResultView({
result: M48SFixedClassDetectorResult; result: M48SFixedClassDetectorResult;
}) { }) {
const selected = result.metrics.candidates.find((candidate) => candidate.selected); const selected = result.metrics.candidates.find((candidate) => candidate.selected);
const load = result.metrics.detectorLoad; const detectorLoad = result.metrics.detectorLoad;
const integrated = result.metrics.integratedWorldState; const integrated = result.metrics.integratedWorldState;
const status = integrated const hardening = result.metrics.runtimeHardening;
const loadEnvelope = result.metrics.loadEnvelope;
const production = loadEnvelope?.scenarios.find((scenario) => scenario.id === "production-10fps");
const reserve = loadEnvelope?.scenarios.find((scenario) => scenario.id === "reserve-12fps");
const limit = loadEnvelope?.scenarios.find((scenario) => scenario.id === "limit-15fps");
const load = loadEnvelope && production && reserve && limit
? { comparison: loadEnvelope, production, reserve, limit }
: null;
const loadAccepted = load?.comparison.loadEnvelopeAccepted === true;
const status = load
? loadAccepted
? "3 × 10 / 12 / 15 FPS прошли; realtime envelope ≥ 15 FPS принят"
: "Capacity ≥ 15 FPS подтверждён; repeatability 10 FPS требует закрытия"
: hardening
? `После hardening: ${hardening.hardened.deliveredWorldStates.toLocaleString("ru-RU")} из ${hardening.hardened.sourceFramesAdmitted.toLocaleString("ru-RU")} world states доставлены`
: integrated
? "Полный RF-DETR reference graph выдержал realtime shadow" ? "Полный RF-DETR reference graph выдержал realtime shadow"
: "RF-DETR-L выдержал detector-only realtime shadow"; : "RF-DETR-L выдержал detector-only realtime shadow";
return ( return (
@@ -29,9 +44,11 @@ export function M48SFixedClassDetectorResultView({
summary={( summary={(
<LaboratorySummary <LaboratorySummary
title="M4.8S · fixed-class semantics риск-объектов" title="M4.8S · fixed-class semantics риск-объектов"
description="Сравнение трёх готовых COCO-детекторов на точных кадрах RAVNOVES00, 30-минутная квалификация RF-DETR-L и полный source-paced прогон RF-DETR → geometry → temporal → motion → rolling map → threat на Worker 006. Статические препятствия остаются в геометрическом контуре; классы используются только там, где меняется ожидаемое поведение." description={loadAccepted
? "RF-DETR-L встроен в полный source-paced граф RF-DETR → geometry → temporal → motion → rolling map → threat. Холодный video decode вынесен до допуска источника, после чего один immutable проход выполнен по три раза на 10, 12 и 15 FPS. LAB показывает худший результат каждой серии, хвосты decode/pacing и каждый из девяти sealed-прогонов."
: "RF-DETR-L встроен в полный source-paced граф RF-DETR → geometry → temporal → motion → rolling map → threat. Исторический immutable проход отдельно измерен на 10, 12 и 15 FPS; незакрытый 10 FPS delivery gate сохранён без постфактум-сдвига thresholds."}
status={status} status={status}
statusTone="success" statusTone={load && !loadAccepted ? "warning" : "success"}
facts={[ facts={[
{ label: "Источник", value: `${rigLabel} RIGHT · raw KB4 · ${result.source.evidenceFrameCount} diagnostic frames` }, { label: "Источник", value: `${rigLabel} RIGHT · raw KB4 · ${result.source.evidenceFrameCount} diagnostic frames` },
{ label: "Сравнение", value: "YOLOX-S · D-FINE-S · RF-DETR-L · единый threshold 0.50" }, { label: "Сравнение", value: "YOLOX-S · D-FINE-S · RF-DETR-L · единый threshold 0.50" },
@@ -39,12 +56,28 @@ export function M48SFixedClassDetectorResultView({
{ label: "Authority", value: "SHADOW ONLY · commands OFF · actuation OFF · production NO" }, { label: "Authority", value: "SHADOW ONLY · commands OFF · actuation OFF · production NO" },
]} ]}
brief={{ brief={{
question: "Можно ли заменить слабую class-семантику YOLOX готовой моделью, не потеряв realtime на предельном Worker с RTX 4090?", question: load
approach: `YOLOX-S, D-FINE-S и RF-DETR-L сравнили на одинаковых ${result.source.evidenceFrameCount} raw-KB4 кадрах с порогом 0.50. RF-DETR-L отдельно квалифицировали ${decimal(load.durationSeconds / 60, 0)} минут, затем встроили в полный reference graph без дополнительного inference-прохода.`, ? "Какой realtime-запас имеет неизменный single-pass RF-DETR graph на Worker 006 и где начинается его вычислительный предел?"
principalResult: integrated : "Можно ли заменить слабую class-семантику YOLOX готовой моделью, не потеряв realtime на предельном Worker с RTX 4090?",
approach: loadAccepted
? "Один и тот же проход из 4489 кадров прогнали строго последовательно девять раз: по три повтора на 10, 12 и 15 FPS. Bounded prefetch на 64 кадра завершался до source admission; sensor timestamps, модель, providers, очереди и single inference не менялись. Пороги и порядок были записаны заранее."
: load
? "Один и тот же проход из 4489 кадров прогнали последовательно на 10, 12 и 15 FPS. Sensor timestamps, модель, graph providers, очереди, prewarm и single inference не менялись; менялся только wall-clock pacing. Пороги delivery/FPS/p95 были записаны до запусков."
: `YOLOX-S, D-FINE-S и RF-DETR-L сравнили на одинаковых ${result.source.evidenceFrameCount} raw-KB4 кадрах с порогом 0.50. RF-DETR-L отдельно квалифицировали ${decimal(result.metrics.detectorLoad.durationSeconds / 60, 0)} минут, затем встроили в полный reference graph без дополнительного inference-прохода.`,
principalResult: loadAccepted && load
? `Все девять прогонов прошли integrity и predeclared target gates. Худший результат: 10 FPS — ${load.production.deliveredWorldStates.toLocaleString("ru-RU")}/${load.production.sourceFramesAdmitted.toLocaleString("ru-RU")}, 12 FPS — ${load.reserve.deliveredWorldStates.toLocaleString("ru-RU")}/${load.reserve.sourceFramesAdmitted.toLocaleString("ru-RU")}, 15 FPS — ${load.limit.deliveredWorldStates.toLocaleString("ru-RU")}/${load.limit.sourceFramesAdmitted.toLocaleString("ru-RU")}. Во всех сериях сохранён один inference на кадр.`
: load
? `12 FPS прошли reserve-gate: ${load.reserve.deliveredWorldStates.toLocaleString("ru-RU")}/${load.reserve.sourceFramesAdmitted.toLocaleString("ru-RU")} доставлено. 15 FPS также прошли: ${load.limit.deliveredWorldStates.toLocaleString("ru-RU")}/${load.limit.sourceFramesAdmitted.toLocaleString("ru-RU")}. Но 10 FPS не добрали заранее заданный delivery ratio: ${load.production.deliveredWorldStates.toLocaleString("ru-RU")}/${load.production.sourceFramesAdmitted.toLocaleString("ru-RU")}.`
: hardening
? `На одинаковых ${hardening.baseline.sourceFramesAdmitted.toLocaleString("ru-RU")} входных кадрах полный граф увеличил доставку с ${hardening.baseline.deliveredWorldStates.toLocaleString("ru-RU")} до ${hardening.hardened.deliveredWorldStates.toLocaleString("ru-RU")} world states, а число вытесненных кадров снизил с ${hardening.baseline.supersededFrames} до ${hardening.hardened.supersededFrames}. Отдельный ${hardening.startup.validationFrames.toLocaleString("ru-RU")}-кадровый прогон подтвердил prewarm до допуска источника.`
: integrated
? `Полный граф доставил ${integrated.deliveredWorldStates.toLocaleString("ru-RU")} world states при ${decimal(integrated.effectiveWorldStateFps, 3)} FPS и p95 ${decimal(integrated.worldStateCompletionAgeP95Ms, 3)} ms; ${integrated.supersededFrames} входных кадров штатно вытеснены latest-wins очередью.` ? `Полный граф доставил ${integrated.deliveredWorldStates.toLocaleString("ru-RU")} world states при ${decimal(integrated.effectiveWorldStateFps, 3)} FPS и p95 ${decimal(integrated.worldStateCompletionAgeP95Ms, 3)} ms; ${integrated.supersededFrames} входных кадров штатно вытеснены latest-wins очередью.`
: `RF-DETR-L выбран из трёх кандидатов и обработал ${load.sourceFramesConsumed.toLocaleString("ru-RU")} кадров detector-only без замен и ошибок.`, : `RF-DETR-L выбран из трёх кандидатов и обработал ${detectorLoad.sourceFramesConsumed.toLocaleString("ru-RU")} кадров detector-only без замен и ошибок.`,
limitation: "Прогон доказывает runtime envelope, а не истинность классов, качество track identity, корректность risk policy или безопасность движения. Production authority и команды отключены.", limitation: loadAccepted
? "Runtime repeatability и capacity полного графа закрыты как минимум до 15 FPS. Это не доказывает истинность классов, track/risk truth, collision safety или physical-live поведение; команды и production authority по-прежнему отключены."
: load
? "Capacity полного графа доказан как минимум до 15 FPS, но исторический production repeatability gate не принят. Истинность классов, collision safety и physical-live не проверялись; команды отключены."
: "Прогон доказывает runtime envelope, а не истинность классов, качество track identity, корректность risk policy или безопасность движения. Production authority и команды отключены.",
}} }}
method={{ method={{
completeness: result.method.completeness, completeness: result.method.completeness,
@@ -73,28 +106,67 @@ export function M48SFixedClassDetectorResultView({
)} )}
result={( result={(
<LaboratoryResultSummary <LaboratoryResultSummary
title={integrated title={hardening
? load
? "Что сравнивать: один graph на 10 / 12 / 15 FPS"
: "Что сравнивать: один и тот же полный прогон до и после hardening"
: integrated
? "Что дал прогон: полный world-state graph проходит realtime envelope" ? "Что дал прогон: полный world-state graph проходит realtime envelope"
: "Что дал прогон: RF-DETR-L проходит detector-only realtime envelope"} : "Что дал прогон: RF-DETR-L проходит detector-only realtime envelope"}
status={status} status={status}
statusTone="success" statusTone={load && !loadAccepted ? "warning" : "success"}
metrics={integrated ? [ metrics={load ? [
{ label: "Effective world-state FPS", value: `${decimal(load.production.effectiveWorldStateFps, 3)} / ${decimal(load.reserve.effectiveWorldStateFps, 3)} / ${decimal(load.limit.effectiveWorldStateFps, 3)}`, hint: "вход 10 / 12 / 15 FPS" },
{ label: "Worst delivered / superseded", value: `${load.production.deliveredWorldStates.toLocaleString("ru-RU")}/${load.production.supersededFrames} · ${load.reserve.deliveredWorldStates.toLocaleString("ru-RU")}/${load.reserve.supersededFrames} · ${load.limit.deliveredWorldStates.toLocaleString("ru-RU")}/${load.limit.supersededFrames}`, hint: "минимальная доставка / максимум вытеснений из трёх повторов" },
{ label: "World-state p95", value: `${decimal(load.production.worldStateCompletionAgeP95Ms, 3)} / ${decimal(load.reserve.worldStateCompletionAgeP95Ms, 3)} / ${decimal(load.limit.worldStateCompletionAgeP95Ms, 3)} ms`, hint: "10 / 12 / 15 FPS" },
{ label: "World-state p99", value: `${decimal(load.production.worldStateCompletionAgeP99Ms, 3)} / ${decimal(load.reserve.worldStateCompletionAgeP99Ms, 3)} / ${decimal(load.limit.worldStateCompletionAgeP99Ms, 3)} ms`, hint: "хвост не растёт вместе с rate" },
{ label: loadAccepted ? "Hot-loop decode p95" : "Decode p95", value: `${decimal(load.production.hotLoopDecodeP95Ms, 3)} / ${decimal(load.reserve.hotLoopDecodeP95Ms, 3)} / ${decimal(load.limit.hotLoopDecodeP95Ms, 3)} ms`, hint: loadAccepted ? "холодный open/decode исключён из realtime" : "исторический decode включает cold tail" },
loadAccepted
? { label: "Prefetch / pacing max", value: `${decimal(Math.max(...load.comparison.scenarios.map((scenario) => scenario.sourcePrefetchPreparationMaximumMs)) / 1000, 2)} s / ${decimal(Math.max(...load.comparison.scenarios.map((scenario) => scenario.sourcePacingLatenessMaximumMs)), 3)} ms`, hint: "prefetch до admission / wall-clock pacing в hot loop" }
: { label: "Decode maximum", value: `${decimal(load.production.hotLoopDecodeMaximumMs, 0)} / ${decimal(load.reserve.hotLoopDecodeMaximumMs, 0)} / ${decimal(load.limit.hotLoopDecodeMaximumMs, 0)} ms`, hint: "редкий cold source/decode tail" },
{ label: "Detector p95", value: `${decimal(load.production.detectorP95Ms, 3)} / ${decimal(load.reserve.detectorP95Ms, 3)} / ${decimal(load.limit.detectorP95Ms, 3)} ms`, hint: "тот же RF-DETR · один inference" },
{ label: "GPU mean", value: `${decimal(load.production.gpuUtilizationMeanPercent)}% / ${decimal(load.reserve.gpuUtilizationMeanPercent)}% / ${decimal(load.limit.gpuUtilizationMeanPercent)}%`, hint: "10 / 12 / 15 FPS · насыщения нет" },
{ label: "VRAM / process RAM peak", value: `${decimal(Math.max(...load.comparison.scenarios.map((scenario) => scenario.gpuMemoryMaximumMib)) / 1024, 2)} / ${decimal(Math.max(...load.comparison.scenarios.map((scenario) => scenario.processPeakRssMib)) / 1024, 2)} GiB`, hint: "общий GPU counter / graph process RSS" },
] : hardening ? [
{ label: "World-state FPS", value: `${decimal(hardening.baseline.effectiveWorldStateFps, 3)} → ${decimal(hardening.hardened.effectiveWorldStateFps, 3)}`, hint: "до → после · target ≥ 9.5 FPS" },
{ label: "Задержка p95", value: `${decimal(hardening.baseline.worldStateCompletionAgeP95Ms, 3)} → ${decimal(hardening.hardened.worldStateCompletionAgeP95Ms, 3)} ms`, hint: `p99 ${decimal(hardening.baseline.worldStateCompletionAgeP99Ms, 3)} → ${decimal(hardening.hardened.worldStateCompletionAgeP99Ms, 3)} ms` },
{ label: "Максимальная задержка", value: `${decimal(hardening.baseline.worldStateCompletionAgeMaximumMs, 3)} → ${decimal(hardening.hardened.worldStateCompletionAgeMaximumMs, 3)} ms`, hint: "полный прогон без prewarm" },
{ label: "Доставлено / вытеснено", value: `${hardening.baseline.deliveredWorldStates.toLocaleString("ru-RU")}/${hardening.baseline.supersededFrames} → ${hardening.hardened.deliveredWorldStates.toLocaleString("ru-RU")}/${hardening.hardened.supersededFrames}`, hint: `${hardening.hardened.sourceFramesAdmitted.toLocaleString("ru-RU")} одинаковых входных кадров` },
{ label: "Rolling max", value: `${decimal(hardening.baseline.rollingMaximumMs, 3)} → ${decimal(hardening.hardened.rollingMaximumMs, 3)} ms`, hint: "убран длинный стоп realtime-контура" },
{ label: "Geometry max", value: `${decimal(hardening.baseline.geometryMaximumMs, 3)} → ${decimal(hardening.hardened.geometryMaximumMs, 3)} ms`, hint: "тот же полный source-paced прогон" },
{ label: "Первый detector frame", value: `${decimal(hardening.startup.baseline.detectorMs, 3)} → ${decimal(hardening.startup.prewarmed.detectorMs, 3)} ms`, hint: `prewarm ${decimal(hardening.startup.prewarmDurationMs, 3)} ms до допуска source` },
{ label: "Первый world state", value: `${decimal(hardening.startup.baseline.worldStateMs, 3)} → ${decimal(hardening.startup.prewarmed.worldStateMs, 3)} ms`, hint: `${hardening.startup.validationFrames.toLocaleString("ru-RU")} кадров · ${hardening.startup.prewarmInferencePasses} warmup inference` },
] : integrated ? [
{ label: "Complete graph", value: `${decimal(integrated.effectiveWorldStateFps, 3)} FPS`, hint: "target ≥ 9.5 FPS · source-paced" }, { label: "Complete graph", value: `${decimal(integrated.effectiveWorldStateFps, 3)} FPS`, hint: "target ≥ 9.5 FPS · source-paced" },
{ label: "World-state age p95", value: `${decimal(integrated.worldStateCompletionAgeP95Ms, 3)} ms`, hint: `p99 ${decimal(integrated.worldStateCompletionAgeP99Ms, 3)} ms · target ≤ 175 ms` }, { label: "World-state age p95", value: `${decimal(integrated.worldStateCompletionAgeP95Ms, 3)} ms`, hint: `p99 ${decimal(integrated.worldStateCompletionAgeP99Ms, 3)} ms · target ≤ 175 ms` },
{ label: "Delivered / superseded", value: `${integrated.deliveredWorldStates.toLocaleString("ru-RU")} / ${integrated.supersededFrames}`, hint: `${integrated.failures} failures · queues ${Math.max(...Object.values(integrated.queueHighWatermarks))}/${integrated.queueCapacity}` }, { label: "Delivered / superseded", value: `${integrated.deliveredWorldStates.toLocaleString("ru-RU")} / ${integrated.supersededFrames}`, hint: `${integrated.failures} failures · queues ${Math.max(...Object.values(integrated.queueHighWatermarks))}/${integrated.queueCapacity}` },
{ label: "GPU / VRAM peak", value: `${decimal(integrated.gpuUtilizationMaximumPercent, 0)}% / ${decimal(integrated.gpuMemoryMaximumMib / 1024)} GiB`, hint: `GPU mean ${decimal(integrated.gpuUtilizationMeanPercent)}% · ${decimal(integrated.gpuPowerMaximumW)} W` }, { label: "GPU / VRAM peak", value: `${decimal(integrated.gpuUtilizationMaximumPercent, 0)}% / ${decimal(integrated.gpuMemoryMaximumMib / 1024)} GiB`, hint: `GPU mean ${decimal(integrated.gpuUtilizationMeanPercent)}% · ${decimal(integrated.gpuPowerMaximumW)} W` },
] : [ ] : [
{ label: "Detector capacity", value: `${decimal(selected?.capacityFps ?? 0)} FPS`, hint: "RF-DETR-L TensorRT/Triton" }, { label: "Detector capacity", value: `${decimal(selected?.capacityFps ?? 0)} FPS`, hint: "RF-DETR-L TensorRT/Triton" },
{ label: "Completion age p95", value: `${decimal(load.completionAgeP95Ms)} ms`, hint: "detector-only · target ≤ 175 ms" }, { label: "Completion age p95", value: `${decimal(detectorLoad.completionAgeP95Ms)} ms`, hint: "detector-only · target ≤ 175 ms" },
{ label: "Consumed / replaced", value: `${load.sourceFramesConsumed.toLocaleString("ru-RU")} / ${load.sourceFrameReplacements}`, hint: `${decimal(load.effectiveConsumedFps, 3)} source FPS · ${load.failures} failures` }, { label: "Consumed / replaced", value: `${detectorLoad.sourceFramesConsumed.toLocaleString("ru-RU")} / ${detectorLoad.sourceFrameReplacements}`, hint: `${decimal(detectorLoad.effectiveConsumedFps, 3)} source FPS · ${detectorLoad.failures} failures` },
{ label: "GPU / VRAM peak", value: `${decimal(load.gpuUtilizationMaximumPercent, 0)}% / ${decimal(load.gpuMemoryMaximumMib / 1024)} GiB`, hint: `GPU mean ${decimal(load.gpuUtilizationMeanPercent)}% · queue ${load.queueMaximumDepth}/${load.queueCapacity}` }, { label: "GPU / VRAM peak", value: `${decimal(detectorLoad.gpuUtilizationMaximumPercent, 0)}% / ${decimal(detectorLoad.gpuMemoryMaximumMib / 1024)} GiB`, hint: `GPU mean ${decimal(detectorLoad.gpuUtilizationMeanPercent)}% · queue ${detectorLoad.queueMaximumDepth}/${detectorLoad.queueCapacity}` },
]} ]}
conclusion={{ conclusion={{
proved: integrated proved: load
? `Все ${load.comparison.repeatCountPerRate * load.comparison.scenarios.length} полных single-pass прогона сохранили bounded queues 2/2, нулевые failed/stale/rejected/unavailable и нулевые дополнительные inference-проходы. Худший 10 FPS delivery — ${(load.production.deliveryRatio * 100).toLocaleString("ru-RU", { maximumFractionDigits: 3 })}%, 12 FPS — ${(load.reserve.deliveryRatio * 100).toLocaleString("ru-RU", { maximumFractionDigits: 3 })}%, 15 FPS — ${(load.limit.deliveryRatio * 100).toLocaleString("ru-RU", { maximumFractionDigits: 3 })}%. Realtime capacity доказана как минимум до 15 FPS без более мощного Worker.`
: hardening
? `На полном сравнительном прогоне с ${hardening.hardened.sourceFramesAdmitted.toLocaleString("ru-RU")} одинаковыми входными кадрами hardening сохранил realtime FPS, доставил на ${(hardening.hardened.deliveredWorldStates - hardening.baseline.deliveredWorldStates).toLocaleString("ru-RU")} world states больше и сократил вытеснения ${hardening.baseline.supersededFrames} → ${hardening.hardened.supersededFrames}. Максимальный стоп rolling stage уменьшился ${decimal(hardening.baseline.rollingMaximumMs, 3)} → ${decimal(hardening.hardened.rollingMaximumMs, 3)} ms. Отдельный prewarm-прогон снизил первый detector frame ${decimal(hardening.startup.baseline.detectorMs, 3)} → ${decimal(hardening.startup.prewarmed.detectorMs, 3)} ms и первый world state ${decimal(hardening.startup.baseline.worldStateMs, 3)} → ${decimal(hardening.startup.prewarmed.worldStateMs, 3)} ms без дополнительного inference на кадрах.`
: integrated
? `На Worker 006 полный граф обработал ${integrated.sourceFramesAdmitted.toLocaleString("ru-RU")} входных кадров, доставил ${integrated.deliveredWorldStates.toLocaleString("ru-RU")} состояний без ошибок, удержал все очереди в пределах ${integrated.queueCapacity} и p95 ${decimal(integrated.worldStateCompletionAgeP95Ms, 3)} ms. Advisory сформировал публикации по семействам: geometry-only (${integrated.advisoryFamilyCounts["generic-obstacle"].toLocaleString("ru-RU")}), люди (${integrated.advisoryFamilyCounts.person.toLocaleString("ru-RU")}), животные (${integrated.advisoryFamilyCounts.animal.toLocaleString("ru-RU")}) и транспорт (${integrated.advisoryFamilyCounts.vehicle.toLocaleString("ru-RU")}); это не количество уникальных физических объектов и не потребовало второго inference.` ? `На Worker 006 полный граф обработал ${integrated.sourceFramesAdmitted.toLocaleString("ru-RU")} входных кадров, доставил ${integrated.deliveredWorldStates.toLocaleString("ru-RU")} состояний без ошибок, удержал все очереди в пределах ${integrated.queueCapacity} и p95 ${decimal(integrated.worldStateCompletionAgeP95Ms, 3)} ms. Advisory сформировал публикации по семействам: geometry-only (${integrated.advisoryFamilyCounts["generic-obstacle"].toLocaleString("ru-RU")}), люди (${integrated.advisoryFamilyCounts.person.toLocaleString("ru-RU")}), животные (${integrated.advisoryFamilyCounts.animal.toLocaleString("ru-RU")}) и транспорт (${integrated.advisoryFamilyCounts.vehicle.toLocaleString("ru-RU")}); это не количество уникальных физических объектов и не потребовало второго inference.`
: `RF-DETR-L ${decimal(load.durationSeconds / 60, 0)} минут устойчиво потреблял source-paced поток около 10 FPS: ${load.sourceFramesConsumed.toLocaleString("ru-RU")} кадров, 0 замен, 0 ошибок, completion-age p95 ${decimal(load.completionAgeP95Ms, 3)} ms.`, : `RF-DETR-L ${decimal(detectorLoad.durationSeconds / 60, 0)} минут устойчиво потреблял source-paced поток около 10 FPS: ${detectorLoad.sourceFramesConsumed.toLocaleString("ru-RU")} кадров, 0 замен, 0 ошибок, completion-age p95 ${decimal(detectorLoad.completionAgeP95Ms, 3)} ms.`,
notProved: "Не доказаны unbiased precision/recall классов, независимое качество track identity и risk policy, поведение planner или collision safety. Кадровые рамки не заменяют геометрическую occupancy-карту.", notProved: loadAccepted
decision: "Сохранить RF-DETR-L как risk-semantic shadow provider полного reference graph. Не классифицировать миллионы статических форм: неизвестное неподвижное препятствие остаётся geometry-owned и объезжается; классы сохраняются для людей, животных и транспорта. Production switch не разрешён.", ? "Не доказаны unbiased precision/recall классов, независимая истинность track identity и risk policy, работа planner, collision safety или physical-live режим. Принятый runtime envelope не включает production/navigation authority."
: load
? `Не закрыта repeatability delivery на production-rate: 10 FPS дали ${(load.production.deliveryRatio * 100).toLocaleString("ru-RU", { maximumFractionDigits: 3 })}% при gate ${(load.production.thresholds.minimumDeliveryRatio * 100).toLocaleString("ru-RU", { maximumFractionDigits: 1 })}%. Исторический v1 результат сохранён; он не переинтерпретируется после появления v2.`
: "Не доказаны unbiased precision/recall классов, независимое качество track identity и risk policy, поведение planner или collision safety. Кадровые рамки не заменяют геометрическую occupancy-карту.",
decision: loadAccepted
? "Сохранить bounded 64-frame prefetch до source admission как runtime baseline. Не усиливать GPU и не добавлять второй детектор: Worker 006 уже держит как минимум 15 FPS. Следующий этап — semantic/track/risk truth и physical-live shadow; production switch не разрешён."
: load
? "Историческое решение v1: не усиливать GPU и не добавлять второй детектор; изолировать cold video decode и повторить sealed 10/12/15 FPS серией прогонов. Результат v2 закрывает этот следующий этап отдельно, не переписывая v1."
: hardening
? "Сохранить разгрузку cyclic GC и обязательный detector prewarm как текущий runtime baseline. Следующий этап — измерять запас realtime при росте сцены и числа семантических объектов, не добавляя второй детектор и не расширяя inference-нагрузку. Production switch не разрешён."
: "Сохранить RF-DETR-L как risk-semantic shadow provider полного reference graph. Не классифицировать миллионы статических форм: неизвестное неподвижное препятствие остаётся geometry-owned и объезжается; классы сохраняются для людей, животных и транспорта. Production switch не разрешён.",
}} }}
/> />
)} )}
@@ -45,6 +45,59 @@ function resultPayload() {
selected: false, selected: false,
...overrides, ...overrides,
}); });
const loadScenario = (id, loadPurpose, rate, delivered, superseded) => ({
id,
load_purpose: loadPurpose,
requested_source_rate_hz: rate,
source_frames_admitted: 4489,
delivered_world_states: delivered,
superseded_frames: superseded,
delivery_ratio: delivered / 4489,
effective_world_state_fps: rate * 0.975,
world_state_completion_age_p95_ms: 75,
world_state_completion_age_p99_ms: 100,
world_state_completion_age_maximum_ms: 400,
hot_loop_decode_p95_ms: 14,
hot_loop_decode_maximum_ms: 42,
preadmission_decode_maximum_ms: 3633,
source_prefetch_preparation_maximum_ms: 4353,
source_pacing_lateness_p95_ms: 1.7,
source_pacing_lateness_maximum_ms: 11,
detector_p95_ms: 40,
detector_maximum_ms: 370,
gpu_utilization_mean_percent: 35,
gpu_utilization_maximum_percent: 85,
gpu_memory_maximum_mib: 9584,
process_peak_rss_mib: 2327,
queue_high_watermarks: { detector: 2, geometry: 2, temporal: 2, rolling: 2, threat: 2 },
queue_capacity: 2,
additional_inference_passes: 0,
repeat_count: 3,
all_repetitions_passed: true,
integrity_gate_passed: true,
operating_target_gate_passed: true,
thresholds: {
minimum_delivery_ratio: rate === 10 ? 0.999 : rate === 12 ? 0.995 : 0.95,
minimum_effective_world_state_fps: rate * 0.95,
maximum_world_state_completion_p95_ms: rate === 10 ? 125 : rate === 12 ? 150 : 175,
},
repetitions: [1, 2, 3].map((repetition) => ({
run_id: `m48s-load-prefetch-v1-${id}-a${repetition}`,
repetition,
delivered_world_states: delivered,
superseded_frames: superseded,
delivery_ratio: delivered / 4489,
effective_world_state_fps: rate * 0.975,
world_state_completion_age_p95_ms: 75,
hot_loop_decode_p95_ms: 14,
hot_loop_decode_maximum_ms: 42,
preadmission_decode_maximum_ms: 3633,
source_pacing_lateness_maximum_ms: 11,
operating_target_gate_passed: true,
result_sha256: "7".repeat(64),
frame_evidence_sha256: "8".repeat(64),
})),
});
return { return {
schema_version: "missioncore.m48s-fixed-class-detector-result-view/v1", schema_version: "missioncore.m48s-fixed-class-detector-result-view/v1",
result_id: resultId, result_id: resultId,
@@ -138,6 +191,55 @@ function resultPayload() {
additional_inference_passes: 0, additional_inference_passes: 0,
failures: 0, failures: 0,
}, },
runtime_hardening: {
schema_version: "missioncore.m48s-runtime-hardening-comparison/v1",
baseline: {
source_frames_admitted: 4489,
delivered_world_states: 4480,
superseded_frames: 9,
effective_world_state_fps: 9.750635,
world_state_completion_age_p95_ms: 76.886564,
world_state_completion_age_p99_ms: 100.661219,
world_state_completion_age_maximum_ms: 794.748644,
rolling_maximum_ms: 762.638263,
geometry_maximum_ms: 350.066302,
additional_inference_passes: 0,
},
hardened: {
source_frames_admitted: 4489,
delivered_world_states: 4488,
superseded_frames: 1,
effective_world_state_fps: 9.821942,
world_state_completion_age_p95_ms: 75.270655,
world_state_completion_age_p99_ms: 94.309605,
world_state_completion_age_maximum_ms: 449.893287,
rolling_maximum_ms: 28.044958,
geometry_maximum_ms: 49.174149,
additional_inference_passes: 0,
},
startup: {
baseline: { detector_ms: 397.356888, world_state_ms: 449.893287 },
prewarmed: { detector_ms: 27.662887, world_state_ms: 60.516957 },
prewarm_duration_ms: 420.721918,
prewarm_inference_passes: 1,
validation_frames: 1000,
},
},
load_envelope: {
schema_version: "missioncore.m48s-load-envelope-comparison/v2",
repeat_count_per_rate: 3,
production_rate_repeatability_passed: true,
reserve_12_fps_passed: true,
limit_15_fps_passed: true,
load_envelope_accepted: true,
compute_capacity_at_least_fps: 15,
bottleneck_interpretation: "cold-video-decode-isolated-before-admission-no-steady-gpu-saturation",
scenarios: [
loadScenario("production-10fps", "production-rate", 10, 4489, 0),
loadScenario("reserve-12fps", "reserve-gate", 12, 4488, 1),
loadScenario("limit-15fps", "limit-discovery", 15, 4488, 1),
],
},
}, },
decision: { decision: {
selected_candidate: "rf-detr", selected_candidate: "rf-detr",
@@ -145,6 +247,11 @@ function resultPayload() {
integrated_world_state_gate_evaluated: true, integrated_world_state_gate_evaluated: true,
integrated_world_state_gate_passed: true, integrated_world_state_gate_passed: true,
detector_replacement_authorized: false, detector_replacement_authorized: false,
load_envelope_evaluated: true,
production_rate_repeatability_passed: true,
reserve_12_fps_passed: true,
limit_15_fps_passed: true,
load_envelope_accepted: true,
production_accepted: false, production_accepted: false,
}, },
limitations: ["No independent semantic ground truth."], limitations: ["No independent semantic ground truth."],
@@ -175,11 +282,71 @@ test("M4.8S result exposes complete graph load without production authority", as
assert.equal(result.metrics.integratedWorldState.deliveredWorldStates, 4481); assert.equal(result.metrics.integratedWorldState.deliveredWorldStates, 4481);
assert.equal(result.metrics.integratedWorldState.worldStateCompletionAgeP95Ms, 74.733648); assert.equal(result.metrics.integratedWorldState.worldStateCompletionAgeP95Ms, 74.733648);
assert.equal(result.metrics.integratedWorldState.additionalInferencePasses, 0); assert.equal(result.metrics.integratedWorldState.additionalInferencePasses, 0);
assert.equal(result.metrics.runtimeHardening.baseline.supersededFrames, 9);
assert.equal(result.metrics.runtimeHardening.hardened.supersededFrames, 1);
assert.equal(result.metrics.runtimeHardening.startup.prewarmed.worldStateMs, 60.516957);
assert.equal(result.metrics.loadEnvelope.scenarios[0].operatingTargetGatePassed, true);
assert.equal(result.metrics.loadEnvelope.scenarios[1].deliveredWorldStates, 4488);
assert.equal(result.metrics.loadEnvelope.scenarios[2].repetitions.length, 3);
assert.equal(result.metrics.loadEnvelope.scenarios[0].hotLoopDecodeP95Ms, 14);
assert.equal(result.metrics.loadEnvelope.computeCapacityAtLeastFps, 15);
assert.equal(result.decision.loadEnvelopeAccepted, true);
assert.equal(result.decision.integratedWorldStateGatePassed, true); assert.equal(result.decision.integratedWorldStateGatePassed, true);
assert.equal(result.decision.productionAccepted, false); assert.equal(result.decision.productionAccepted, false);
assert.equal(result.authority.navigationOrSafetyAccepted, false); assert.equal(result.authority.navigationOrSafetyAccepted, false);
}); });
test("M4.8S keeps the immutable v1 load-envelope result readable", async () => {
const payload = resultPayload();
payload.metrics.load_envelope = {
schema_version: "missioncore.m48s-load-envelope-comparison/v1",
production_rate_repeatability_passed: false,
reserve_12_fps_passed: true,
limit_15_fps_passed: true,
load_envelope_accepted: false,
compute_capacity_at_least_fps: 15,
bottleneck_interpretation: "rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation",
scenarios: payload.metrics.load_envelope.scenarios.map((scenario, index) => ({
id: scenario.id,
load_purpose: scenario.load_purpose,
requested_source_rate_hz: scenario.requested_source_rate_hz,
source_frames_admitted: scenario.source_frames_admitted,
delivered_world_states: index === 0 ? 4483 : scenario.delivered_world_states,
superseded_frames: index === 0 ? 6 : scenario.superseded_frames,
delivery_ratio: index === 0 ? 4483 / 4489 : scenario.delivery_ratio,
effective_world_state_fps: scenario.effective_world_state_fps,
world_state_completion_age_p95_ms: scenario.world_state_completion_age_p95_ms,
world_state_completion_age_p99_ms: scenario.world_state_completion_age_p99_ms,
world_state_completion_age_maximum_ms: scenario.world_state_completion_age_maximum_ms,
decode_p95_ms: scenario.hot_loop_decode_p95_ms,
decode_maximum_ms: 3200,
detector_p95_ms: scenario.detector_p95_ms,
detector_maximum_ms: scenario.detector_maximum_ms,
gpu_utilization_mean_percent: scenario.gpu_utilization_mean_percent,
gpu_utilization_maximum_percent: scenario.gpu_utilization_maximum_percent,
gpu_memory_maximum_mib: scenario.gpu_memory_maximum_mib,
process_peak_rss_mib: scenario.process_peak_rss_mib,
queue_high_watermarks: scenario.queue_high_watermarks,
queue_capacity: 2,
additional_inference_passes: 0,
integrity_gate_passed: true,
operating_target_gate_passed: index !== 0,
thresholds: scenario.thresholds,
frame_evidence_sha256: "8".repeat(64),
})),
};
payload.decision.production_rate_repeatability_passed = false;
payload.decision.load_envelope_accepted = false;
const result = await fetchM48SFixedClassDetectorResult(resultId, {
fetcher: async () => response(payload),
});
assert.equal(result.metrics.loadEnvelope.repeatCountPerRate, 1);
assert.equal(result.metrics.loadEnvelope.loadEnvelopeAccepted, false);
assert.equal(result.metrics.loadEnvelope.scenarios[0].operatingTargetGatePassed, false);
});
test("M4.8S frame binds exact camera endpoint and risk-only boxes", async () => { test("M4.8S frame binds exact camera endpoint and risk-only boxes", async () => {
const frame = await fetchM48SFixedClassDetectorFrame(resultId, "000253", { const frame = await fetchM48SFixedClassDetectorFrame(resultId, "000253", {
fetcher: async () => response({ fetcher: async () => response({
+1 -1
View File
@@ -249,7 +249,7 @@
}, },
{ {
"catalog_id": "m48s-fixed-class-detector", "catalog_id": "m48s-fixed-class-detector",
"evidence_id": "m48s-fixed-class-detector-lab-d1bac05a9e43d407b0f931105cc0e84183ef9ff37666911f03c41486beeb7ef9", "evidence_id": "m48s-fixed-class-detector-lab-4a901d811f53734540337f8d1f2c01666539aa0a9b7786f26d4f1f2e08cc858a",
"signal": "progress", "signal": "progress",
"lifecycle": "current", "lifecycle": "current",
"visual_evidence": "available" "visual_evidence": "available"
@@ -0,0 +1,113 @@
{
"schema_version": "missioncore.m48s-load-envelope-profile/v2",
"profile_id": "m48s-rf-detr-reference-graph-load-envelope-prefetch/v1",
"decision_question": "Does bounded recorded-image prefetch remove decode-startup interference and make the 10, 12 and 15 FPS complete-graph results repeatable on Worker 006?",
"hypothesis": "A 64-frame bounded decoder prefetch completed before source admission isolates cold video open and initial decode from the realtime hot loop. Three complete repetitions at each rate will retain exact terminal accounting, satisfy the predeclared targets and attribute any residual tail to hot-loop decode, source pacing or graph completion.",
"source": {
"source_id": "RAVNOVES00",
"source_profile_id": "m4-ravnoves00-recorded-realtime/v1",
"frame_count": 4489,
"graph_id": "reference-perception-graph/v2",
"detector_provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0",
"sensor_timestamps_preserved": true
},
"controlled_change_from_v1": {
"change": "bounded 64-frame recorded-image decode prefetch before source admission plus source pacing attribution",
"unchanged": [
"source frames and source timestamps",
"RF-DETR TensorRT engine and threshold",
"geometry, temporal, motion, rolling-map and threat providers",
"latest-wins queue capacities",
"single inference pass per admitted detector frame",
"detector prewarm and cyclic-GC hot-loop policy"
]
},
"series_control": {
"repeat_count_per_rate": 3,
"execution_order": [
"production-10fps-a1",
"reserve-12fps-a1",
"limit-15fps-a1",
"production-10fps-a2",
"reserve-12fps-a2",
"limit-15fps-a2",
"production-10fps-a3",
"reserve-12fps-a3",
"limit-15fps-a3"
],
"worker": "DESKTOP-OPJ8J04 / Worker 006",
"execution": "strictly sequential; one graph container and one candidate Triton container per run"
},
"scenarios": [
{
"id": "production-10fps",
"run_ids": [
"m48s-load-prefetch-v1-production-10fps-a1",
"m48s-load-prefetch-v1-production-10fps-a2",
"m48s-load-prefetch-v1-production-10fps-a3"
],
"load_purpose": "production-rate",
"source_rate_hz": 10.0,
"thresholds": {
"minimum_delivery_ratio": 0.999,
"minimum_effective_world_state_fps": 9.5,
"maximum_world_state_completion_p95_ms": 125.0
},
"required_for_reserve_decision": true
},
{
"id": "reserve-12fps",
"run_ids": [
"m48s-load-prefetch-v1-reserve-12fps-a1",
"m48s-load-prefetch-v1-reserve-12fps-a2",
"m48s-load-prefetch-v1-reserve-12fps-a3"
],
"load_purpose": "reserve-gate",
"source_rate_hz": 12.0,
"thresholds": {
"minimum_delivery_ratio": 0.995,
"minimum_effective_world_state_fps": 11.4,
"maximum_world_state_completion_p95_ms": 150.0
},
"required_for_reserve_decision": true
},
{
"id": "limit-15fps",
"run_ids": [
"m48s-load-prefetch-v1-limit-15fps-a1",
"m48s-load-prefetch-v1-limit-15fps-a2",
"m48s-load-prefetch-v1-limit-15fps-a3"
],
"load_purpose": "limit-discovery",
"source_rate_hz": 15.0,
"thresholds": {
"minimum_delivery_ratio": 0.95,
"minimum_effective_world_state_fps": 14.25,
"maximum_world_state_completion_p95_ms": 175.0
},
"required_for_reserve_decision": false
}
],
"common_integrity_gates": [
"closed terminal accounting",
"zero failed, stale, rejected or unavailable frames",
"all queue high-watermarks at or below capacity two",
"complete single-pass pipeline timing",
"detector prewarm before source admission",
"source prefetch ready before source admission",
"source pacing attribution for every admitted frame",
"cyclic GC disabled only during the hot loop and restored after",
"all command, actuation, navigation and safety authority remains false"
],
"exit_decision": {
"repeatability_accepted_when": "All three repetitions at 10 and 12 FPS pass every integrity and operating-target gate; 15 FPS remains limit evidence and must pass integrity.",
"optimization_scope_when_rejected": "Optimize only the phase identified by preadmission/hot-loop decode and source-pacing attribution; do not add another detector or inference pass."
},
"authority": {
"candidate_accepted": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false,
"production_accepted": false
}
}
@@ -0,0 +1,89 @@
{
"schema_version": "missioncore.m48s-load-envelope-profile/v1",
"profile_id": "m48s-rf-detr-reference-graph-load-envelope/v1",
"decision_question": "Can the unchanged single-pass RF-DETR reference graph sustain the recorded production rate and 20 percent reserve on Worker 006, and where does its bounded latest-wins limit begin?",
"hypothesis": "The hardened single-pass graph sustains 10 FPS and 12 FPS without unbounded queues, failed frames, a second inference pass, or more than the predeclared delivery loss; 15 FPS is measured only to locate the limit.",
"source": {
"source_id": "RAVNOVES00",
"source_profile_id": "m4-ravnoves00-recorded-realtime/v1",
"frame_count": 4489,
"graph_id": "reference-perception-graph/v2",
"detector_provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0",
"sensor_timestamps_preserved": true
},
"controlled_change": {
"only_variable": "wall_clock_source_rate_hz",
"unchanged": [
"source frames and source timestamps",
"RF-DETR TensorRT engine and threshold",
"geometry, temporal, motion, rolling-map and threat providers",
"latest-wins queue capacities",
"single inference pass per admitted detector frame",
"detector prewarm and cyclic-GC hot-loop policy"
]
},
"scenarios": [
{
"id": "production-10fps",
"run_id": "m48s-load-envelope-v1-production-10fps-a1",
"load_purpose": "production-rate",
"source_rate_hz": 10.0,
"thresholds": {
"minimum_delivery_ratio": 0.999,
"minimum_effective_world_state_fps": 9.5,
"maximum_world_state_completion_p95_ms": 125.0
},
"required_for_reserve_decision": true
},
{
"id": "reserve-12fps",
"run_id": "m48s-load-envelope-v1-reserve-12fps-a1",
"load_purpose": "reserve-gate",
"source_rate_hz": 12.0,
"thresholds": {
"minimum_delivery_ratio": 0.995,
"minimum_effective_world_state_fps": 11.4,
"maximum_world_state_completion_p95_ms": 150.0
},
"required_for_reserve_decision": true
},
{
"id": "limit-15fps",
"run_id": "m48s-load-envelope-v1-limit-15fps-a1",
"load_purpose": "limit-discovery",
"source_rate_hz": 15.0,
"thresholds": {
"minimum_delivery_ratio": 0.95,
"minimum_effective_world_state_fps": 14.25,
"maximum_world_state_completion_p95_ms": 175.0
},
"required_for_reserve_decision": false
}
],
"common_integrity_gates": [
"closed terminal accounting",
"zero failed, stale, rejected or unavailable frames",
"all queue high-watermarks at or below capacity two",
"complete single-pass pipeline timing",
"detector prewarm before source admission",
"cyclic GC disabled only during the hot loop and restored after",
"all command, actuation, navigation and safety authority remains false"
],
"visual_evidence": {
"binding": "reuse exact M4.8S full camera plus LiDAR plus 3D/PLAN timeline",
"reason": "Only wall-clock pacing changes; source frames, sensor timestamps and graph semantics remain immutable.",
"independent_ground_truth": false
},
"exit_decision": {
"reserve_accepted_when": "Both production-10fps and reserve-12fps pass every integrity and operating target gate.",
"optimization_scope_when_rejected": "Optimize only the measured bottleneck stage; do not add another detector or inference pass.",
"limit_discovery_interpretation": "The 15 FPS result is valid evidence when integrity gates pass even if its operating target gate fails."
},
"authority": {
"candidate_accepted": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false,
"production_accepted": false
}
}
@@ -0,0 +1,116 @@
# Mission Core backend lifecycle audit — 2026-08-25
## Outcome
The observed outage was operator-tool induced, not an unexplained Python crash,
GPU out-of-memory event or host memory leak. A Codex session repeatedly used
forced LaunchAgent restarts while the LAB camera/timeline endpoints still had
active work. The final forced termination left the old Uvicorn child draining
while a replacement tried to acquire the same singleton service lease.
The failure class is a **process-lifecycle termination leak**: the old process
remained alive beyond the restart command's assumption. It is not evidence of
unbounded heap growth.
## Evidence and causal chain
The private Codex rollout journal
`~/.codex/sessions/2026/08/25/rollout-2026-08-25T01-08-27-01a035d1-587d-7112-9506-7cc801c2863c.jsonl`
for the active 2026-08-25 task records
`launchctl kickstart -k` against `com.nodedc.mission-core.local` at 14:40,
15:38, 15:44, 16:22, 16:32, 17:14, 17:29, 17:31 and 17:42 Moscow time. At
17:15 it additionally records `SIGTERM`, a ten-second wait, then `SIGKILL`
against the exact old process before another kickstart.
The user-visible 17:09 Moscow-time outage occurred after the 16:32 forced
restart and before the later 17:15 TERM/KILL recovery attempt. This ordering
rules out the later kill as the start of that outage while still attributes the
failure window to the same repeated forced-restart sequence.
The LaunchAgent evidence showed 139 historical runs and last exit code 143
(`SIGTERM`), with no jetsam/OOM record. The application log showed Uvicorn
entering graceful shutdown and waiting for connections/background tasks while
M4.8S LAB camera/timeline requests were open. Replacement processes reported
that Mission Core was already starting or stopping because the old child still
held `.runtime/mission-core/.serve.lock`.
The complete causal chain was:
```text
Codex forced launchctl restart
-> SIGTERM reached the uv/Uvicorn generation
-> Uvicorn waited without a configured graceful-shutdown deadline
-> old child retained the singleton flock during active LAB work
-> launchd observed its wrapper transition and attempted a replacement
-> replacement failed closed on the singleton lease
-> port 8000 remained unavailable until the old generation was killed
```
The Codex agent caused the outage. The backend did not spontaneously fall over.
## Code and runtime audit
The audit covered Python service startup/shutdown, ASGI lifespan cleanup,
thread joins, subprocess calls, the LaunchAgent declaration and operator
restart paths under `src/k1link` and `scripts`.
| Finding | Severity | State | Resolution |
| --- | --- | --- | --- |
| MC-LIFE-001: Uvicorn graceful drain had no deadline | P0 | fixed | `timeout_graceful_shutdown=10` |
| MC-LIFE-002: PID-only launchd supervision could not detect a live unhealthy service | P0 | fixed | exact-health self-watchdog, three-failure gate, TERM then KILL |
| MC-LIFE-003: forced restart did not prove old label/process release | P0 | fixed | SHA-bound plan/apply, full `bootout` disappearance wait, health acceptance and rollback |
| MC-LIFE-004: dependency resolution could mutate or delay a recovery launch | P1 | fixed | canonical launcher uses `uv run --no-sync` |
| MC-LIFE-004A: launcher parent exit could leave its Python child generation | P1 | fixed | `AbandonProcessGroup=false` makes launchd own the complete group |
| MC-LIFE-005: ASGI/plugin close functions can individually block | P1 | bounded externally | Uvicorn 10 s, watchdog 12 s escalation and launchd 20 s deadline bound the whole generation |
| MC-LIFE-006: self-health state previously had no separate durable evidence | P1 | fixed | private rotating JSONL watchdog journal |
| MC-LIFE-007: three unbounded joins exist in the offline E33 qualification runner | P2 | isolated | daemon-only offline worker path; not imported or executed by the backend lifecycle |
| MC-LIFE-008: artifact-build/guardrail scripts contain subprocess calls without local deadlines | P3 | isolated | developer/CI paths only; not service-reachable and cannot hold port 8000 |
All subprocess calls reachable through the backend probes and compute-network
control paths already carry explicit deadlines. Service-owned joins found in
the active backend, camera, viewer, preparation, LiDAR-shadow, simulation and
protocol lifecycles are bounded. The remaining no-timeout calls identified by
the syntax scan are offline artifact/qualification tooling, not request or
lifespan paths.
The direct `.venv/bin/k1link` LaunchAgent entrypoint was also tested and
rejected by macOS with `EPERM` while reading `.venv/pyvenv.cfg` below the
`Downloads` privacy boundary. The apply tool restored the previous plist and
health. The accepted declaration therefore retains the already-authorized
Homebrew `uv` boundary and disables syncing; it does not weaken macOS privacy
controls.
## Recovery and resume semantics
`KeepAlive` now restores a dead process. The self-watchdog converts a
live-but-unhealthy event-loop/application generation into a bounded process
exit so `KeepAlive` can act. Startup reconstructs read-only plugin runtimes,
catalogs and background reconciliation from durable artifacts.
Auto-resume is intentionally selective:
- idempotent read/catalog/preparation reconciliation restarts automatically;
- browser WebSockets reconnect to the new generation;
- interrupted physical acquisition is reconciled or marked interrupted from
durable ledgers;
- physical commands, acquisition continuation, navigation and actuation are
never silently resumed.
This distinction prevents availability recovery from becoming an authority
escalation.
## Qualification evidence
Focused lifecycle/perception tests passed before deployment. The installed
LaunchAgent accepted exact health with watchdog enabled and a 20-second exit
deadline. A controlled termination of the complete Mission Core process group
produced a new LaunchAgent PID and exact health in 26.588 seconds without
manual intervention. A separate `SIGKILL` crash injection changed PID/PGID
`42895` to PID `42942`, restored exact health in 22.996 seconds and left no old
group residue.
The local qualification does not make the current Mac LaunchAgent an onboard
deployment artifact. The eventual onboard init declaration must independently
prove boot start, crash restart, health-hang restart, power-loss recovery,
bounded shutdown, single-generation fencing and fail-closed physical-state
reconciliation.
@@ -0,0 +1,110 @@
# Mission Core local service recovery
## Scope
This runbook owns the single local Mission Core backend at
`http://127.0.0.1:8000`. It covers startup, health supervision, bounded
shutdown and recovery on the current macOS operator station. It does not grant
physical acquisition, actuation, navigation or safety authority.
The canonical LaunchAgent label is:
```text
com.nodedc.mission-core.local
```
Do not start a second backend on another port. Do not use
`launchctl kickstart -k` for this service: it combines termination and restart
without proving that the old process group has released the singleton lease.
## Recovery contract
The installed LaunchAgent and application form one bounded recovery ladder:
1. `launchd` starts one `uv run --no-sync k1link serve` process group with
`RunAtLoad=true`, `KeepAlive=true`, `AbandonProcessGroup=false` and a
five-second throttle.
2. Mission Core starts a private self-health thread after acquiring the
singleton backend lease.
3. After the 45-second cold-start grace, three consecutive failed exact
`/api/health` probes request `SIGTERM` for the complete process group.
4. Uvicorn stops accepting work and has ten seconds to drain active requests
and ASGI lifespan work.
5. If the service remains alive, the watchdog escalates to `SIGKILL` after
twelve seconds. `launchd` also owns a 20-second exit deadline.
6. `launchd` starts a fresh process group. Startup reconstructs plugin
runtimes and the recording-preparation reconciler from durable state.
The watchdog journal is private, bounded and rotated:
```text
.runtime/mission-core/service-watchdog.jsonl
.runtime/mission-core/service-watchdog.jsonl.1
```
Physical operations are deliberately not resumed from an assumed state.
Interrupted preparation and catalog work is reconciled from durable evidence;
physical acquisition, commands and actuation remain fail-closed and require a
new confirmed authority transition.
## Plan and apply
Always plan from the repository root before changing the installed agent:
```bash
uv run python scripts/manage_mission_core_launch_agent.py plan \
--repository-root "$PWD"
```
Copy the exact `current_sha256` and `desired_sha256` from that output into the
apply command:
```bash
uv run python scripts/manage_mission_core_launch_agent.py apply \
--repository-root "$PWD" \
--expected-current-sha256 <current-sha256> \
--expected-desired-sha256 <desired-sha256>
```
Apply writes a mode-0600 backup below
`.runtime/mission-core/launch-agent-backups`, atomically replaces the plist,
waits until `bootout` has fully removed the old label, bootstraps the new
declaration and accepts only the exact Mission Core health document. Any
failure restores the previous plist and repeats the same health acceptance.
Read-only status:
```bash
uv run python scripts/manage_mission_core_launch_agent.py status \
--repository-root "$PWD"
```
## Acceptance after recovery
The service is recovered only when all of the following are true:
- `launchctl print gui/$(id -u)/com.nodedc.mission-core.local` reports
`state = running`;
- the arguments include `uv run --no-sync k1link serve`;
- the environment contains `MISSIONCORE_SERVICE_WATCHDOG => 1`;
- the launchd exit timeout is 20 seconds;
- `GET http://127.0.0.1:8000/api/health` returns HTTP 200 with
`ok=true`, `status=ok` and
`service=mission-core-control-plane`;
- the watchdog journal contains `watchdog-started` for the current child PID;
- there is only one `uv` parent and one Mission Core Python child in their
exact process group.
## 2026-08-25 recovery qualification
The reviewed declaration SHA-256 was
`80fca5ec6bdab21f11a544d8dbee65a35f73a6c34752c6a48e9a1181e5da256a`.
A controlled `SIGTERM` of the exact Mission Core process group changed the
LaunchAgent PID from `40866` to `40957`; exact health recovered automatically
in `26.588` seconds without a manual start. After the explicit
`AbandonProcessGroup=false` fence was installed, a controlled `SIGKILL` of PID
and PGID `42895` produced a new LaunchAgent PID `42942` and exact health in
`22.996` seconds, with no old process-group residue. This is local
operator-station evidence only. An onboard Linux deployment must express the
same contract in its init system and pass its own power-loss, crash, hang and
durable-state reconciliation qualification.
@@ -47,14 +47,14 @@ from k1link.perception.m48s_reference_graph_runtime import (
from k1link.perception.motion import ClassIndependentMotionEstimator from k1link.perception.motion import ClassIndependentMotionEstimator
from k1link.perception.object_understanding import AdvisoryResponse from k1link.perception.object_understanding import AdvisoryResponse
from k1link.perception.providers import SourcePacket from k1link.perception.providers import SourcePacket
from k1link.perception.recorded_source import DecodedFrameTiming from k1link.perception.recorded_source import DecodedFrameTiming, SourcePacingTiming
from k1link.perception.reference_graph_runtime import ReferenceGraphRuntimePaths from k1link.perception.reference_graph_runtime import ReferenceGraphRuntimePaths
from k1link.perception.rolling_map import RollingLocalObstacleMapProvider from k1link.perception.rolling_map import RollingLocalObstacleMapProvider
from k1link.perception.temporal import BoundedSpatialTemporalProvider from k1link.perception.temporal import BoundedSpatialTemporalProvider
SCHEMA_VERSION: Final = "missioncore.m48s-reference-graph-shadow-load/v3" SCHEMA_VERSION: Final = "missioncore.m48s-reference-graph-shadow-load/v5"
FRAME_EVIDENCE_SCHEMA: Final = "missioncore.m48s-reference-graph-frame-evidence/v1" FRAME_EVIDENCE_SCHEMA: Final = "missioncore.m48s-reference-graph-frame-evidence/v1"
PIPELINE_TIMING_SCHEMA: Final = "missioncore.m48s-frame-pipeline-timing/v0" PIPELINE_TIMING_SCHEMA: Final = "missioncore.m48s-frame-pipeline-timing/v1"
GC_POLICY_SCHEMA: Final = "missioncore.cyclic-gc-hot-loop-policy/v0" GC_POLICY_SCHEMA: Final = "missioncore.cyclic-gc-hot-loop-policy/v0"
AUTHORITY: Final = { AUTHORITY: Final = {
"ground_truth": False, "ground_truth": False,
@@ -63,6 +63,7 @@ AUTHORITY: Final = {
"actuation_allowed": False, "actuation_allowed": False,
"navigation_or_safety_accepted": False, "navigation_or_safety_accepted": False,
} }
LOAD_PURPOSES: Final = ("production-rate", "reserve-gate", "limit-discovery")
class GpuTelemetry: class GpuTelemetry:
@@ -227,14 +228,23 @@ class FrameTimingStore:
def __init__(self) -> None: def __init__(self) -> None:
self._lock = threading.Lock() self._lock = threading.Lock()
self._decode: dict[int, int] = {} self._decode: dict[int, DecodedFrameTiming] = {}
self._pacing: dict[int, SourcePacingTiming] = {}
self._all_decode: list[DecodedFrameTiming] = []
self._all_pacing: list[SourcePacingTiming] = []
self._detector: dict[int, DetectorFrameTiming] = {} self._detector: dict[int, DetectorFrameTiming] = {}
self._providers: dict[int, dict[str, int]] = defaultdict(dict) self._providers: dict[int, dict[str, int]] = defaultdict(dict)
self._delivered: list[dict[str, object]] = [] self._delivered: list[dict[str, object]] = []
def observe_decode(self, timing: DecodedFrameTiming) -> None: def observe_decode(self, timing: DecodedFrameTiming) -> None:
with self._lock: with self._lock:
self._decode[timing.sequence] = timing.duration_ns self._decode[timing.sequence] = timing
self._all_decode.append(timing)
def observe_pacing(self, timing: SourcePacingTiming) -> None:
with self._lock:
self._pacing[timing.sequence] = timing
self._all_pacing.append(timing)
def observe_detector(self, timing: DetectorFrameTiming) -> None: def observe_detector(self, timing: DetectorFrameTiming) -> None:
with self._lock: with self._lock:
@@ -256,7 +266,8 @@ class FrameTimingStore:
) -> dict[str, object]: ) -> dict[str, object]:
with self._lock: with self._lock:
try: try:
decode_ns = self._decode.pop(sequence) decode = self._decode.pop(sequence)
pacing = self._pacing.pop(sequence)
detector = self._detector.pop(sequence) detector = self._detector.pop(sequence)
providers = self._providers.pop(sequence) providers = self._providers.pop(sequence)
except KeyError as exc: except KeyError as exc:
@@ -271,13 +282,19 @@ class FrameTimingStore:
document = { document = {
"schema_version": PIPELINE_TIMING_SCHEMA, "schema_version": PIPELINE_TIMING_SCHEMA,
"sequence": sequence, "sequence": sequence,
"decode_duration_ns": decode_ns, "decode_duration_ns": decode.duration_ns,
"decode_phase": decode.phase.value,
"source_pacing": {
"scheduled_monotonic_ns": pacing.scheduled_monotonic_ns,
"emitted_monotonic_ns": pacing.emitted_monotonic_ns,
"lateness_ns": pacing.lateness_ns,
},
"detector": detector.to_dict(), "detector": detector.to_dict(),
"providers": dict(sorted(providers.items())), "providers": dict(sorted(providers.items())),
"graph_admission_to_delivery_ns": admission_to_delivery_ns, "graph_admission_to_delivery_ns": admission_to_delivery_ns,
"graph_attributed_provider_ns": attributed_graph_ns, "graph_attributed_provider_ns": attributed_graph_ns,
"graph_unattributed_ns": unattributed_ns, "graph_unattributed_ns": unattributed_ns,
"decode_to_delivery_processing_ns": decode_ns + admission_to_delivery_ns, "decode_to_delivery_processing_ns": decode.duration_ns + admission_to_delivery_ns,
} }
with self._lock: with self._lock:
self._delivered.append(document) self._delivered.append(document)
@@ -287,6 +304,14 @@ class FrameTimingStore:
with self._lock: with self._lock:
return tuple(self._delivered) return tuple(self._delivered)
def all_decode(self) -> tuple[DecodedFrameTiming, ...]:
with self._lock:
return tuple(self._all_decode)
def all_pacing(self) -> tuple[SourcePacingTiming, ...]:
with self._lock:
return tuple(self._all_pacing)
class TimedProviderProxy: class TimedProviderProxy:
"""Record one provider's actual call duration without another inference pass.""" """Record one provider's actual call duration without another inference pass."""
@@ -359,6 +384,13 @@ def main() -> int:
parser.add_argument("--triton-origin", default="http://127.0.0.1:8000") parser.add_argument("--triton-origin", default="http://127.0.0.1:8000")
parser.add_argument("--loops", type=int, default=1) parser.add_argument("--loops", type=int, default=1)
parser.add_argument("--maximum-frames", type=int) parser.add_argument("--maximum-frames", type=int)
parser.add_argument("--source-rate-hz", type=float)
parser.add_argument("--minimum-delivery-ratio", type=float, default=0.0)
parser.add_argument("--minimum-effective-world-state-fps", type=float, default=9.5)
parser.add_argument("--maximum-world-state-completion-p95-ms", type=float, default=175.0)
parser.add_argument("--load-purpose", choices=LOAD_PURPOSES, default="production-rate")
parser.add_argument("--runtime-artifact-sha256", required=True)
parser.add_argument("--runner-sha256", required=True)
parser.add_argument("--telemetry-interval-seconds", type=float, default=1.0) parser.add_argument("--telemetry-interval-seconds", type=float, default=1.0)
parser.add_argument("--output", type=Path, required=True) parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--progress", type=Path, required=True) parser.add_argument("--progress", type=Path, required=True)
@@ -370,6 +402,21 @@ def main() -> int:
raise RuntimeError("maximum frame count must be positive") raise RuntimeError("maximum frame count must be positive")
if arguments.telemetry_interval_seconds <= 0: if arguments.telemetry_interval_seconds <= 0:
raise RuntimeError("telemetry interval must be positive") raise RuntimeError("telemetry interval must be positive")
if arguments.source_rate_hz is not None and (
not np.isfinite(arguments.source_rate_hz) or arguments.source_rate_hz <= 0
):
raise RuntimeError("source rate must be positive and finite")
if not 0.0 <= arguments.minimum_delivery_ratio <= 1.0:
raise RuntimeError("minimum delivery ratio must be between zero and one")
if arguments.minimum_effective_world_state_fps <= 0:
raise RuntimeError("minimum effective world-state FPS must be positive")
if arguments.maximum_world_state_completion_p95_ms <= 0:
raise RuntimeError("maximum completion p95 must be positive")
for digest in (arguments.runtime_artifact_sha256, arguments.runner_sha256):
if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest):
raise RuntimeError("runtime and runner SHA-256 values must be lowercase hex")
if _sha256(Path(__file__)) != arguments.runner_sha256:
raise RuntimeError("runner SHA-256 changed")
output = arguments.output.absolute() output = arguments.output.absolute()
progress = arguments.progress.absolute() progress = arguments.progress.absolute()
frame_ledger = arguments.frame_ledger.absolute() frame_ledger = arguments.frame_ledger.absolute()
@@ -397,6 +444,8 @@ def main() -> int:
map_output_ages_ms: list[float] = [] map_output_ages_ms: list[float] = []
all_deliveries: list[DeliveredFrame] = [] all_deliveries: list[DeliveredFrame] = []
all_pipeline_timings: list[dict[str, object]] = [] all_pipeline_timings: list[dict[str, object]] = []
all_decode_timings: list[DecodedFrameTiming] = []
all_pacing_timings: list[SourcePacingTiming] = []
with ( with (
progress.open("x", encoding="utf-8") as progress_stream, progress.open("x", encoding="utf-8") as progress_stream,
frame_ledger.open("x", encoding="utf-8") as frame_ledger_stream, frame_ledger.open("x", encoding="utf-8") as frame_ledger_stream,
@@ -424,8 +473,10 @@ def main() -> int:
timing_store, timing_store,
), ),
decode_timing_observer=timing_store.observe_decode, decode_timing_observer=timing_store.observe_decode,
source_pacing_observer=timing_store.observe_pacing,
detector_timing_observer=timing_store.observe_detector, detector_timing_observer=timing_store.observe_detector,
maximum_frames=arguments.maximum_frames, maximum_frames=arguments.maximum_frames,
source_rate_hz=arguments.source_rate_hz,
) as runtime: ) as runtime:
for stage_id, attribute in ( for stage_id, attribute in (
("geometry", "geometry"), ("geometry", "geometry"),
@@ -445,8 +496,10 @@ def main() -> int:
), ),
) )
detector_warmup = runtime.warm_up_detector() detector_warmup = runtime.warm_up_detector()
source_prefetch = runtime.prepare_source()
gc_policy = CyclicGcHotLoopPolicy() gc_policy = CyclicGcHotLoopPolicy()
with gc_policy: with gc_policy:
runtime.mark_source_admission_started()
loop_started_ns = time.monotonic_ns() loop_started_ns = time.monotonic_ns()
result = runtime.graph.run() result = runtime.graph.run()
loop_completed_ns = time.monotonic_ns() loop_completed_ns = time.monotonic_ns()
@@ -499,6 +552,7 @@ def main() -> int:
setup_seconds=setup_seconds, setup_seconds=setup_seconds,
gc_policy=gc_policy.to_dict(), gc_policy=gc_policy.to_dict(),
detector_warmup=detector_warmup, detector_warmup=detector_warmup,
source_prefetch=asdict(source_prefetch),
) )
loop_documents.append(loop_document) loop_documents.append(loop_document)
completion_ages_ms.extend(value / 1_000_000.0 for value in loop_completion_ages_ns) completion_ages_ms.extend(value / 1_000_000.0 for value in loop_completion_ages_ns)
@@ -507,6 +561,8 @@ def main() -> int:
) )
all_deliveries.extend(result.deliveries) all_deliveries.extend(result.deliveries)
all_pipeline_timings.extend(loop_pipeline_timings) all_pipeline_timings.extend(loop_pipeline_timings)
all_decode_timings.extend(timing_store.all_decode())
all_pacing_timings.extend(timing_store.all_pacing())
frame_ledger_stream.flush() frame_ledger_stream.flush()
progress_row = { progress_row = {
"loop": loop_index + 1, "loop": loop_index + 1,
@@ -546,7 +602,9 @@ def main() -> int:
identity = _identity_metrics(all_deliveries) identity = _identity_metrics(all_deliveries)
semantic = _semantic_metrics(all_deliveries, advisories) semantic = _semantic_metrics(all_deliveries, advisories)
world_state_fps = delivered / processing_wall_seconds world_state_fps = delivered / processing_wall_seconds
checks = { delivery_ratio = delivered / admitted if admitted else 0.0
completion_distribution = _distribution(completion_ages_ms)
integrity_checks = {
"loop_count_completed": len(loop_documents) == arguments.loops, "loop_count_completed": len(loop_documents) == arguments.loops,
"graph_stopped_cleanly": all( "graph_stopped_cleanly": all(
loop["state"] == GraphState.STOPPED.value for loop in loop_documents loop["state"] == GraphState.STOPPED.value for loop in loop_documents
@@ -561,10 +619,6 @@ def main() -> int:
TerminalOutcomeType.UNAVAILABLE, TerminalOutcomeType.UNAVAILABLE,
) )
), ),
"minimum_world_state_fps": world_state_fps >= 9.5,
"maximum_world_state_completion_p95_ms": (
_distribution(completion_ages_ms)["p95"] <= 175.0
),
"bounded_latest_wins_queues": all(value <= 2 for value in queue_high_watermarks.values()), "bounded_latest_wins_queues": all(value <= 2 for value in queue_high_watermarks.values()),
"temporal_identity_reuse_observed": cast( "temporal_identity_reuse_observed": cast(
int, int,
@@ -586,8 +640,27 @@ def main() -> int:
and cast(dict[str, object], loop["detector_warmup"])["inference_passes"] == 1 and cast(dict[str, object], loop["detector_warmup"])["inference_passes"] == 1
for loop in loop_documents for loop in loop_documents
), ),
"source_prefetch_completed_before_source_admission": all(
cast(dict[str, object], loop["source_prefetch"])["buffered_frames"]
== cast(dict[str, object], loop["source_prefetch"])["ready_frames"]
for loop in loop_documents
),
"source_pacing_attribution_complete": len(all_pacing_timings) == admitted,
"authority_remains_false": all(value is False for value in AUTHORITY.values()), "authority_remains_false": all(value is False for value in AUTHORITY.values()),
} }
operating_target_checks = {
"minimum_delivery_ratio": delivery_ratio >= arguments.minimum_delivery_ratio,
"minimum_world_state_fps": (
world_state_fps >= arguments.minimum_effective_world_state_fps
),
"maximum_world_state_completion_p95_ms": (
completion_distribution["p95"]
<= arguments.maximum_world_state_completion_p95_ms
),
}
checks = {**integrity_checks, **operating_target_checks}
evidence_integrity_gate_passed = all(integrity_checks.values())
operating_target_gate_passed = all(operating_target_checks.values())
integrated_runtime_gate_passed = all(checks.values()) integrated_runtime_gate_passed = all(checks.values())
document = { document = {
"schema_version": SCHEMA_VERSION, "schema_version": SCHEMA_VERSION,
@@ -595,20 +668,28 @@ def main() -> int:
"source_id": "RAVNOVES00", "source_id": "RAVNOVES00",
"loops": arguments.loops, "loops": arguments.loops,
"maximum_frames_per_loop": arguments.maximum_frames, "maximum_frames_per_loop": arguments.maximum_frames,
"requested_rate_hz": arguments.source_rate_hz,
"pacing_contract": "wall-clock-scaled-source-timestamps-immutable/v1",
}, },
"identity": { "identity": {
"worker_id": "worker-006", "worker_id": "worker-006",
"graph_id": "reference-perception-graph/v2", "graph_id": "reference-perception-graph/v2",
"detector_provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0", "detector_provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0",
"inputs": _input_digests(paths, arguments.detector_profile), "inputs": _input_digests(paths, arguments.detector_profile),
"runtime_artifact_sha256": arguments.runtime_artifact_sha256,
"runner_sha256": arguments.runner_sha256,
}, },
"execution": { "execution": {
"run_mode": GraphRunMode.SOURCE_PACED_LATEST_WINS.value, "run_mode": GraphRunMode.SOURCE_PACED_LATEST_WINS.value,
"load_purpose": arguments.load_purpose,
"requested_source_rate_hz": arguments.source_rate_hz,
"source_timestamps_preserved": True,
"wall_seconds": round(wall_seconds, 6), "wall_seconds": round(wall_seconds, 6),
"source_processing_wall_seconds": round(processing_wall_seconds, 6), "source_processing_wall_seconds": round(processing_wall_seconds, 6),
"admitted_frames": admitted, "admitted_frames": admitted,
"delivered_world_states": delivered, "delivered_world_states": delivered,
"effective_world_state_fps": round(world_state_fps, 6), "effective_world_state_fps": round(world_state_fps, 6),
"delivery_ratio": round(delivery_ratio, 9),
"terminal_outcomes": dict(sorted(accounting.items())), "terminal_outcomes": dict(sorted(accounting.items())),
"queue_high_watermarks": queue_high_watermarks, "queue_high_watermarks": queue_high_watermarks,
"loops": loop_documents, "loops": loop_documents,
@@ -624,17 +705,32 @@ def main() -> int:
}, },
}, },
"metrics": { "metrics": {
"world_state_completion_age_ms": _distribution(completion_ages_ms), "world_state_completion_age_ms": completion_distribution,
"local_obstacle_map_output_age_ms": _distribution(map_output_ages_ms), "local_obstacle_map_output_age_ms": _distribution(map_output_ages_ms),
"identity_continuity": identity, "identity_continuity": identity,
"semantic_advisory": semantic, "semantic_advisory": semantic,
"pipeline_timing": _pipeline_timing_metrics(all_pipeline_timings), "pipeline_timing": _pipeline_timing_metrics(all_pipeline_timings),
"source_decode": _source_decode_metrics(all_decode_timings),
"source_pacing": _source_pacing_metrics(all_pacing_timings),
"python_gc": _gc_telemetry_summary(gc_telemetry.events), "python_gc": _gc_telemetry_summary(gc_telemetry.events),
"gpu": _telemetry_summary(gpu.samples), "gpu": _telemetry_summary(gpu.samples),
"process_peak_rss_before_mib": round(rss_before_kib / 1024.0, 6), "process_peak_rss_before_mib": round(rss_before_kib / 1024.0, 6),
"process_peak_rss_after_mib": round(rss_after_kib / 1024.0, 6), "process_peak_rss_after_mib": round(rss_after_kib / 1024.0, 6),
}, },
"checks": checks, "checks": checks,
"integrity_checks": integrity_checks,
"operating_target_checks": operating_target_checks,
"predeclared_thresholds": {
"minimum_delivery_ratio": arguments.minimum_delivery_ratio,
"minimum_effective_world_state_fps": (
arguments.minimum_effective_world_state_fps
),
"maximum_world_state_completion_p95_ms": (
arguments.maximum_world_state_completion_p95_ms
),
},
"evidence_integrity_gate_passed": evidence_integrity_gate_passed,
"operating_target_gate_passed": operating_target_gate_passed,
"integrated_runtime_gate_passed": integrated_runtime_gate_passed, "integrated_runtime_gate_passed": integrated_runtime_gate_passed,
"independent_track_identity_quality_evaluated": False, "independent_track_identity_quality_evaluated": False,
"independent_risk_policy_quality_evaluated": False, "independent_risk_policy_quality_evaluated": False,
@@ -647,7 +743,10 @@ def main() -> int:
output.write_bytes(_canonical_json(document) + b"\n") output.write_bytes(_canonical_json(document) + b"\n")
print(output) print(output)
print(json.dumps(checks, indent=2, sort_keys=True)) print(json.dumps(checks, indent=2, sort_keys=True))
return 0 if integrated_runtime_gate_passed else 2 discovery_completed = (
arguments.load_purpose == "limit-discovery" and evidence_integrity_gate_passed
)
return 0 if integrated_runtime_gate_passed or discovery_completed else 2
def _record_completion_age( def _record_completion_age(
@@ -703,6 +802,7 @@ def _loop_document(
setup_seconds: float, setup_seconds: float,
gc_policy: dict[str, object], gc_policy: dict[str, object],
detector_warmup: DetectorWarmupSnapshot, detector_warmup: DetectorWarmupSnapshot,
source_prefetch: dict[str, object],
) -> dict[str, object]: ) -> dict[str, object]:
outcomes = Counter(item.outcome.value for item in result.terminal_outcomes) outcomes = Counter(item.outcome.value for item in result.terminal_outcomes)
outcome_stages = Counter( outcome_stages = Counter(
@@ -715,6 +815,7 @@ def _loop_document(
"setup_seconds": round(setup_seconds, 6), "setup_seconds": round(setup_seconds, 6),
"cyclic_gc_hot_loop": gc_policy, "cyclic_gc_hot_loop": gc_policy,
"detector_warmup": asdict(detector_warmup), "detector_warmup": asdict(detector_warmup),
"source_prefetch": source_prefetch,
"admitted_count": result.admitted_count, "admitted_count": result.admitted_count,
"delivered_count": len(result.deliveries), "delivered_count": len(result.deliveries),
"effective_world_state_fps": round(len(result.deliveries) / wall_seconds, 6), "effective_world_state_fps": round(len(result.deliveries) / wall_seconds, 6),
@@ -840,6 +941,8 @@ def _pipeline_timing_metrics(
"decode_to_delivery_processing_ns", "decode_to_delivery_processing_ns",
) )
top_level_values: dict[str, list[float]] = {key: [] for key in top_level_fields} top_level_values: dict[str, list[float]] = {key: [] for key in top_level_fields}
decode_by_phase: dict[str, list[float]] = defaultdict(list)
delivered_pacing_lateness_ms: list[float] = []
for document in documents: for document in documents:
detector = cast(Mapping[str, int], document["detector"]) detector = cast(Mapping[str, int], document["detector"])
providers = cast(Mapping[str, int], document["providers"]) providers = cast(Mapping[str, int], document["providers"])
@@ -849,6 +952,11 @@ def _pipeline_timing_metrics(
provider_values[key].append(providers[key] / 1_000_000.0) provider_values[key].append(providers[key] / 1_000_000.0)
for key in top_level_fields: for key in top_level_fields:
top_level_values[key].append(cast(int, document[key]) / 1_000_000.0) top_level_values[key].append(cast(int, document[key]) / 1_000_000.0)
decode_by_phase[cast(str, document["decode_phase"])].append(
cast(int, document["decode_duration_ns"]) / 1_000_000.0
)
source_pacing = cast(Mapping[str, int], document["source_pacing"])
delivered_pacing_lateness_ms.append(source_pacing["lateness_ns"] / 1_000_000.0)
maximum = max( maximum = max(
documents, documents,
key=lambda document: cast(int, document["graph_admission_to_delivery_ns"]), key=lambda document: cast(int, document["graph_admission_to_delivery_ns"]),
@@ -857,6 +965,12 @@ def _pipeline_timing_metrics(
return { return {
"sample_count": len(documents), "sample_count": len(documents),
"decode_duration_ms": _distribution(top_level_values["decode_duration_ns"]), "decode_duration_ms": _distribution(top_level_values["decode_duration_ns"]),
"decode_duration_by_phase_ms": {
phase: _distribution(values) for phase, values in sorted(decode_by_phase.items())
},
"delivered_source_pacing_lateness_ms": _distribution(
delivered_pacing_lateness_ms
),
"detector_ms": { "detector_ms": {
key.removesuffix("_duration_ns"): _distribution(values) key.removesuffix("_duration_ns"): _distribution(values)
for key, values in detector_values.items() for key, values in detector_values.items()
@@ -877,6 +991,50 @@ def _pipeline_timing_metrics(
} }
def _source_decode_metrics(samples: list[DecodedFrameTiming]) -> dict[str, object]:
by_phase: dict[str, list[float]] = defaultdict(list)
for sample in samples:
by_phase[sample.phase.value].append(sample.duration_ns / 1_000_000.0)
return {
"sample_count": len(samples),
"phase_counts": {
phase: len(values) for phase, values in sorted(by_phase.items())
},
"duration_by_phase_ms": {
phase: _distribution(values) for phase, values in sorted(by_phase.items())
},
}
def _source_pacing_metrics(samples: list[SourcePacingTiming]) -> dict[str, object]:
ordered = sorted(samples, key=lambda sample: sample.sequence)
lateness_ms = [sample.lateness_ns / 1_000_000.0 for sample in ordered]
scheduled_intervals_ms = [
(current.scheduled_monotonic_ns - previous.scheduled_monotonic_ns) / 1_000_000.0
for previous, current in zip(ordered, ordered[1:], strict=False)
]
emitted_intervals_ms = [
(current.emitted_monotonic_ns - previous.emitted_monotonic_ns) / 1_000_000.0
for previous, current in zip(ordered, ordered[1:], strict=False)
]
catch_up_emissions = sum(
emitted < scheduled * 0.5
for scheduled, emitted in zip(
scheduled_intervals_ms,
emitted_intervals_ms,
strict=True,
)
if scheduled > 0
)
return {
"sample_count": len(ordered),
"lateness_ms": _distribution(lateness_ms),
"scheduled_interval_ms": _distribution(scheduled_intervals_ms),
"emitted_interval_ms": _distribution(emitted_intervals_ms),
"catch_up_emission_count": catch_up_emissions,
}
def _telemetry_summary(samples: list[dict[str, float]]) -> dict[str, Any]: def _telemetry_summary(samples: list[dict[str, float]]) -> dict[str, Any]:
result: dict[str, Any] = {"sample_count": len(samples)} result: dict[str, Any] = {"sample_count": len(samples)}
for key in ( for key in (
@@ -12,6 +12,16 @@ param(
[int]$Loops = 1, [int]$Loops = 1,
[ValidateRange(1, 4489)] [ValidateRange(1, 4489)]
[int]$MaximumFrames = 120, [int]$MaximumFrames = 120,
[ValidateRange(1.0, 120.0)]
[double]$SourceRateHz = 10.0,
[ValidateRange(0.0, 1.0)]
[double]$MinimumDeliveryRatio = 0.999,
[ValidateRange(0.1, 120.0)]
[double]$MinimumEffectiveWorldStateFps = 9.5,
[ValidateRange(1.0, 10000.0)]
[double]$MaximumWorldStateCompletionP95Ms = 125.0,
[ValidateSet("production-rate", "reserve-gate", "limit-discovery")]
[string]$LoadPurpose = "production-rate",
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m48s-reference-graph-shadow" [string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m48s-reference-graph-shadow"
) )
@@ -103,6 +113,7 @@ $runner = Get-Item -LiteralPath (
if ($runner.PSIsContainer -or ($runner.Attributes -band [IO.FileAttributes]::ReparsePoint)) { if ($runner.PSIsContainer -or ($runner.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
throw "M48S graph runner must be a regular file" throw "M48S graph runner must be a regular file"
} }
$runnerSha256 = Get-Sha256 $runner.FullName
$experimentRoot = Resolve-DDirectory ( $experimentRoot = Resolve-DDirectory (
"D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z" "D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z"
@@ -253,6 +264,13 @@ try {
"--valid-fov-mask", "/source/mask.png", "--valid-fov-mask", "/source/mask.png",
"--triton-origin", "http://127.0.0.1:8000", "--triton-origin", "http://127.0.0.1:8000",
"--loops", ([string]$Loops), "--loops", ([string]$Loops),
"--source-rate-hz", ([string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $SourceRateHz)),
"--minimum-delivery-ratio", ([string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $MinimumDeliveryRatio)),
"--minimum-effective-world-state-fps", ([string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $MinimumEffectiveWorldStateFps)),
"--maximum-world-state-completion-p95-ms", ([string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $MaximumWorldStateCompletionP95Ms)),
"--load-purpose", $LoadPurpose,
"--runtime-artifact-sha256", $ExpectedWheelSha256,
"--runner-sha256", $runnerSha256,
"--output", "/output/result.json", "--output", "/output/result.json",
"--progress", "/output/progress.jsonl", "--progress", "/output/progress.jsonl",
"--frame-ledger", "/output/frames.jsonl" "--frame-ledger", "/output/frames.jsonl"
+221
View File
@@ -0,0 +1,221 @@
#!/usr/bin/env python3
"""Plan/apply the user-owned Mission Core LaunchAgent with rollback acceptance."""
from __future__ import annotations
import argparse
import hashlib
import http.client
import json
import os
import subprocess
import tempfile
import time
from contextlib import suppress
from pathlib import Path
from k1link.local_service_launchd import (
MISSION_CORE_LAUNCH_AGENT_LABEL,
MissionCoreLaunchAgentError,
plan_mission_core_launch_agent,
)
_LAUNCHCTL_TRANSITION_TIMEOUT_SECONDS = 30.0
_LAUNCHCTL_TRANSITION_POLL_SECONDS = 0.1
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=("plan", "apply", "status"))
parser.add_argument("--repository-root", type=Path, required=True)
parser.add_argument(
"--agent-path",
type=Path,
default=Path.home()
/ "Library/LaunchAgents/com.nodedc.mission-core.local.plist",
)
parser.add_argument("--expected-current-sha256")
parser.add_argument("--expected-desired-sha256")
arguments = parser.parse_args()
if arguments.action == "status":
print(json.dumps(_status(), sort_keys=True))
return 0
plan = plan_mission_core_launch_agent(
repository_root=arguments.repository_root,
agent_path=arguments.agent_path,
)
if arguments.action == "plan":
print(json.dumps(plan.to_dict(), indent=2, sort_keys=True))
return 0
if (
arguments.expected_current_sha256 != plan.current_sha256
or arguments.expected_desired_sha256 != plan.desired_sha256
):
raise MissionCoreLaunchAgentError("launch agent plan changed before apply")
previous = plan.agent_path.read_bytes()
backup_root = (
arguments.repository_root.expanduser().resolve()
/ ".runtime/mission-core/launch-agent-backups"
)
backup_root.mkdir(mode=0o700, parents=True, exist_ok=True)
backup = backup_root / f"{plan.current_sha256}.plist"
if backup.exists() and hashlib.sha256(backup.read_bytes()).hexdigest() != plan.current_sha256:
raise MissionCoreLaunchAgentError("launch agent backup identity collision")
if not backup.exists():
_write_atomic(backup, previous)
_write_atomic(plan.agent_path, plan.desired_payload)
try:
_reload_launch_agent(plan.agent_path)
accepted = _wait_for_health(60.0)
if not accepted:
raise MissionCoreLaunchAgentError("reloaded Mission Core did not become healthy")
except BaseException:
_write_atomic(plan.agent_path, previous)
_reload_launch_agent(plan.agent_path)
if not _wait_for_health(60.0):
raise MissionCoreLaunchAgentError(
"launch agent apply failed and rollback did not recover health"
) from None
raise
result = {
**plan.to_dict(),
"applied": True,
"backup_path": str(backup),
"health_accepted": True,
"status": _status(),
}
print(json.dumps(result, indent=2, sort_keys=True))
return 0
def _write_atomic(path: Path, payload: bytes) -> None:
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
try:
os.fchmod(descriptor, 0o600)
with os.fdopen(descriptor, "wb", closefd=True) as stream:
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, path)
except BaseException:
with suppress(OSError):
os.close(descriptor)
with suppress(FileNotFoundError):
os.unlink(temporary)
raise
def _reload_launch_agent(path: Path) -> None:
"""Reload only after launchd proves the previous job is fully absent.
``launchctl bootout`` can return while the job is still represented by an
``xpcproxy`` transition. An immediate bootstrap is then rejected even
though the plist is valid. Waiting on the exact label avoids that race and
keeps apply/rollback symmetric.
"""
domain = f"gui/{os.getuid()}"
target = f"{domain}/{MISSION_CORE_LAUNCH_AGENT_LABEL}"
bootout = subprocess.run(
["launchctl", "bootout", f"{domain}/{MISSION_CORE_LAUNCH_AGENT_LABEL}"],
check=False,
capture_output=True,
text=True,
timeout=30.0,
)
if bootout.returncode != 0 and _launch_agent_loaded(target):
raise MissionCoreLaunchAgentError(
f"launchctl bootout failed with exit code {bootout.returncode}"
)
_wait_until_launch_agent_unloaded(target)
completed = subprocess.run(
["launchctl", "bootstrap", domain, str(path)],
check=False,
capture_output=True,
text=True,
timeout=30.0,
)
if completed.returncode != 0:
raise MissionCoreLaunchAgentError(
"launchctl bootstrap rejected the Mission Core agent "
f"with exit code {completed.returncode}"
)
def _launch_agent_loaded(target: str) -> bool:
completed = subprocess.run(
["launchctl", "print", target],
check=False,
capture_output=True,
text=True,
timeout=5.0,
)
return completed.returncode == 0
def _wait_until_launch_agent_unloaded(target: str) -> None:
deadline = time.monotonic() + _LAUNCHCTL_TRANSITION_TIMEOUT_SECONDS
while time.monotonic() < deadline:
if not _launch_agent_loaded(target):
return
time.sleep(_LAUNCHCTL_TRANSITION_POLL_SECONDS)
raise MissionCoreLaunchAgentError(
"Mission Core launch agent did not finish bootout before the deadline"
)
def _wait_for_health(timeout_seconds: float) -> bool:
deadline = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
if _healthy():
return True
time.sleep(0.25)
return False
def _healthy() -> bool:
connection = http.client.HTTPConnection("127.0.0.1", 8000, timeout=1.0)
try:
connection.request("GET", "/api/health", headers={"Connection": "close"})
response = connection.getresponse()
payload = response.read(64 * 1024 + 1)
except (OSError, TimeoutError, http.client.HTTPException):
return False
finally:
connection.close()
if response.status != 200 or len(payload) > 64 * 1024:
return False
try:
document = json.loads(payload)
except (UnicodeDecodeError, json.JSONDecodeError):
return False
return bool(
isinstance(document, dict)
and document.get("ok") is True
and document.get("status") == "ok"
and document.get("service") == "mission-core-control-plane"
)
def _status() -> dict[str, object]:
completed = subprocess.run(
[
"launchctl",
"print",
f"gui/{os.getuid()}/{MISSION_CORE_LAUNCH_AGENT_LABEL}",
],
check=False,
capture_output=True,
text=True,
timeout=5.0,
)
return {
"label": MISSION_CORE_LAUNCH_AGENT_LABEL,
"launchd_loaded": completed.returncode == 0,
"health_ok": _healthy(),
}
if __name__ == "__main__":
raise SystemExit(main())
+18 -6
View File
@@ -81,6 +81,7 @@ from k1link.device_plugins.xgrids_k1.protocol.application_authority import (
) )
from k1link.device_plugins.xgrids_k1.usb.snapshot import snapshot as usb_snapshot from k1link.device_plugins.xgrids_k1.usb.snapshot import snapshot as usb_snapshot
from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials
from k1link.service_watchdog import MissionCoreSelfWatchdog, watchdog_enabled
from k1link.sessions import SessionIntegrityError from k1link.sessions import SessionIntegrityError
app = typer.Typer( app = typer.Typer(
@@ -116,6 +117,7 @@ app.add_typer(artifact_app, name="artifact")
_CANONICAL_MISSION_CORE_PORT = 8000 _CANONICAL_MISSION_CORE_PORT = 8000
_MISSION_CORE_SERVE_LOCK_FILENAME = ".serve.lock" _MISSION_CORE_SERVE_LOCK_FILENAME = ".serve.lock"
_MISSION_CORE_GRACEFUL_SHUTDOWN_SECONDS = 10
class _MissionCoreServeLeaseError(RuntimeError): class _MissionCoreServeLeaseError(RuntimeError):
@@ -1166,13 +1168,23 @@ def serve_console(
f"NODEDC MISSION CORE: http://127.0.0.1:{_CANONICAL_MISSION_CORE_PORT}" f"NODEDC MISSION CORE: http://127.0.0.1:{_CANONICAL_MISSION_CORE_PORT}"
) )
console.print("The credential endpoint is bound to this Mac only.") console.print("The credential endpoint is bound to this Mac only.")
uvicorn.run( watchdog = (
"k1link.web.app:app", MissionCoreSelfWatchdog(repository_root) if watchdog_enabled() else None
host="127.0.0.1",
port=_CANONICAL_MISSION_CORE_PORT,
log_level="info",
access_log=True,
) )
if watchdog is not None:
watchdog.start()
try:
uvicorn.run(
"k1link.web.app:app",
host="127.0.0.1",
port=_CANONICAL_MISSION_CORE_PORT,
log_level="info",
access_log=True,
timeout_graceful_shutdown=_MISSION_CORE_GRACEFUL_SHUTDOWN_SECONDS,
)
finally:
if watchdog is not None:
watchdog.stop()
def _print_existing_mission_core(port: int) -> None: def _print_existing_mission_core(port: int) -> None:
@@ -21,6 +21,8 @@ LAB_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-lab/v1"
CATALOG_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame-catalog/v1" CATALOG_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame-catalog/v1"
FRAME_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame/v1" FRAME_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame/v1"
REPORT_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-report/v1" REPORT_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-report/v1"
RUNTIME_HARDENING_SCHEMA: Final = "missioncore.m48s-runtime-hardening-comparison/v1"
LOAD_ENVELOPE_SCHEMA: Final = "missioncore.m48s-load-envelope-comparison/v2"
METHOD_SCHEMA: Final = "missioncore.laboratory-method/v1" METHOD_SCHEMA: Final = "missioncore.laboratory-method/v1"
RESULT_PREFIX: Final = "m48s-fixed-class-detector-lab-" RESULT_PREFIX: Final = "m48s-fixed-class-detector-lab-"
TOURNAMENT_ID: Final = ( TOURNAMENT_ID: Final = (
@@ -35,10 +37,154 @@ REFERENCE_GRAPH_ID: Final = (
"e8da7a521768daba0ead1a6e4803871ce3a85f91a7d8ee36c5719ac10433e791" "e8da7a521768daba0ead1a6e4803871ce3a85f91a7d8ee36c5719ac10433e791"
) )
REFERENCE_GRAPH_REPLAY_ID: Final = ( REFERENCE_GRAPH_REPLAY_ID: Final = (
"m48s-reference-graph-replay-" "m48s-reference-graph-replay-16d69d610c22e6f42071b8378cd75dfa6b95db4ceb800f9c7508fa3673504478"
"16d69d610c22e6f42071b8378cd75dfa6b95db4ceb800f9c7508fa3673504478"
) )
INTEGRATED_STATUS: Final = "complete-reference-graph-shadow-passed-production-not-authorized" INTEGRATED_STATUS: Final = "complete-reference-graph-shadow-passed-production-not-authorized"
RUNTIME_HARDENING_RUNS: Final = {
"baseline": {
"directory": "m48s-pipeline-timing-c51761b5",
"schema_version": "missioncore.m48s-reference-graph-shadow-load/v1",
"result_sha256": "277f01a1b6499f08f6fdb65485296c9a6ca214a4f89deaca51d1dc9b00a28528",
"frames_sha256": "c870814159f362e3080ad72a44aab31e1210d5b2d2fd5da8390c05dcd30fbcb8",
"admitted_count": 4489,
"frame_count": 4480,
},
"hardened": {
"directory": "m48s-gc-bounded-477886d",
"schema_version": "missioncore.m48s-reference-graph-shadow-load/v2",
"result_sha256": "13e4423a89b0c1cc78b0219bf5842c42503951ef1372a6996818b3ed8ad3d3ed",
"frames_sha256": "1508da7570fc27aeed010a04ff5eadb6e4a8dedd31e9c617bc021c505a88513d",
"admitted_count": 4489,
"frame_count": 4488,
},
"prewarmed": {
"directory": "m48s-prewarm-84624ae",
"schema_version": "missioncore.m48s-reference-graph-shadow-load/v3",
"result_sha256": "5881f86ac01dc4e8886c4daef5bcd9d3510d50d7f4144c0341e0144ebb7015b1",
"frames_sha256": "67c041beee642f3ff8591c75c80a9bb56c29ad3d63168b38df19ecc475c99d55",
"admitted_count": 1000,
"frame_count": 1000,
},
}
LOAD_ENVELOPE_RUNS: Final = {
"production-10fps-a1": {
"scenario_id": "production-10fps",
"repetition": 1,
"run_id": "m48s-load-prefetch-v1-production-10fps-a1",
"load_purpose": "production-rate",
"source_rate_hz": 10.0,
"result_sha256": "0cf8b682b3972d434e9e10a6eb68a5abc2c1005d731cd61374de951f13d45282",
"frames_sha256": "40d02cb59cb3531b764eface2e5f8b8dbf1d1e4ac83801746a97306ac8363910",
"delivered": 4489,
"superseded": 0,
"operating_target_gate_passed": True,
},
"reserve-12fps-a1": {
"scenario_id": "reserve-12fps",
"repetition": 1,
"run_id": "m48s-load-prefetch-v1-reserve-12fps-a1",
"load_purpose": "reserve-gate",
"source_rate_hz": 12.0,
"result_sha256": "70e0a623624ed162387960867d3615139e452ff30a6ba8daebe6965f39d76f58",
"frames_sha256": "d54187c1b56e43f06498d2deac26374120a36cfd113c3b8f3e4dc170280353aa",
"delivered": 4488,
"superseded": 1,
"operating_target_gate_passed": True,
},
"limit-15fps-a1": {
"scenario_id": "limit-15fps",
"repetition": 1,
"run_id": "m48s-load-prefetch-v1-limit-15fps-a1",
"load_purpose": "limit-discovery",
"source_rate_hz": 15.0,
"result_sha256": "d97ebdcc3bd70dc50284c6482fef614b7ec76180749c32b45c31d5e7e6667133",
"frames_sha256": "b1b79205d266585b0d545bb5a02e87c957759774aeacab2f2d5290cb5d735bf7",
"delivered": 4488,
"superseded": 1,
"operating_target_gate_passed": True,
},
"production-10fps-a2": {
"scenario_id": "production-10fps",
"repetition": 2,
"run_id": "m48s-load-prefetch-v1-production-10fps-a2",
"load_purpose": "production-rate",
"source_rate_hz": 10.0,
"result_sha256": "c98c052b759cecd390e6e79d9eac223e45c96e532bd974d9961d5904945fbfe8",
"frames_sha256": "8574b98aee54ea25e31fc80cbcb7720683bdb36116dc3f2606aed4b0316b8cfa",
"delivered": 4489,
"superseded": 0,
"operating_target_gate_passed": True,
},
"reserve-12fps-a2": {
"scenario_id": "reserve-12fps",
"repetition": 2,
"run_id": "m48s-load-prefetch-v1-reserve-12fps-a2",
"load_purpose": "reserve-gate",
"source_rate_hz": 12.0,
"result_sha256": "16cc2cb0fd9a56707910dfb397021613eb57b1a1d6259b52d14ec38a1e348eb6",
"frames_sha256": "cabd11a2a363b702e2a345198de12408079532ebaa8a095bcfa458acf763325f",
"delivered": 4489,
"superseded": 0,
"operating_target_gate_passed": True,
},
"limit-15fps-a2": {
"scenario_id": "limit-15fps",
"repetition": 2,
"run_id": "m48s-load-prefetch-v1-limit-15fps-a2",
"load_purpose": "limit-discovery",
"source_rate_hz": 15.0,
"result_sha256": "bf23b09b6874f506170448484d1ca2886d8abeb951f9536e810ed24991a0efa6",
"frames_sha256": "fe3b37bde43fe92d92e5da4004509ef87aa85b25700cf8d9d5c84817e13b2422",
"delivered": 4488,
"superseded": 1,
"operating_target_gate_passed": True,
},
"production-10fps-a3": {
"scenario_id": "production-10fps",
"repetition": 3,
"run_id": "m48s-load-prefetch-v1-production-10fps-a3",
"load_purpose": "production-rate",
"source_rate_hz": 10.0,
"result_sha256": "ebb81d69c16b32aa6f2237a3a5e183def58dcb422b1b82fa2621d340cc1b6fcc",
"frames_sha256": "2668b936d274a273dbe440dade55716f686af507aaa4f1a02803396accfe240d",
"delivered": 4489,
"superseded": 0,
"operating_target_gate_passed": True,
},
"reserve-12fps-a3": {
"scenario_id": "reserve-12fps",
"repetition": 3,
"run_id": "m48s-load-prefetch-v1-reserve-12fps-a3",
"load_purpose": "reserve-gate",
"source_rate_hz": 12.0,
"result_sha256": "655d3c0b6c7210f12cfb3a9edf84da5851459afa7303c9732b879e454ac05dc2",
"frames_sha256": "419e8862f0da0e54a3b82f581230ab72874bf52841f62a4e49e1905ed5a77520",
"delivered": 4485,
"superseded": 4,
"operating_target_gate_passed": True,
},
"limit-15fps-a3": {
"scenario_id": "limit-15fps",
"repetition": 3,
"run_id": "m48s-load-prefetch-v1-limit-15fps-a3",
"load_purpose": "limit-discovery",
"source_rate_hz": 15.0,
"result_sha256": "e844183dcee5982f389c9aa847cd98cec78fb81e0b3920e606063740eb35f421",
"frames_sha256": "477cffcecbaf607c8c68979b5425f84963d07f1be76917d2a21925f290bb2736",
"delivered": 4489,
"superseded": 0,
"operating_target_gate_passed": True,
},
}
LOAD_ENVELOPE_PROFILE_SHA256: Final = (
"f0c9467d71622085b31f650d89b70ad5107fa49842f1bc828ccc2c83f85628a3"
)
LOAD_RUNTIME_ARTIFACT_SHA256: Final = (
"9929b735fd139369812f2d5c67149f7737cbf3d3f34076b590e9e9672666bf03"
)
LOAD_RUNNER_SHA256: Final = (
"d3ca51b8500b681307d3a9974f57ae72cdbf04267b5bba26e7ffb091ddd9df31"
)
YOLOX_ID: Final = ( YOLOX_ID: Final = (
"m48s-yolox-all-coco-shadow-7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06" "m48s-yolox-all-coco-shadow-7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06"
) )
@@ -115,6 +261,24 @@ def build_m48s_fixed_class_detector_lab(
reference_graph_replay_path = reference_graph_replay_root / "manifest.json" reference_graph_replay_path = reference_graph_replay_root / "manifest.json"
reference_graph_replay_worker_path = reference_graph_replay_root / "worker-result.json" reference_graph_replay_worker_path = reference_graph_replay_root / "worker-result.json"
reference_graph_replay_frames_path = reference_graph_replay_root / "frames.jsonl" reference_graph_replay_frames_path = reference_graph_replay_root / "frames.jsonl"
hardening_root = runtime / "reference-graph-replay-results"
hardening_paths = {
name: {
"result": hardening_root / str(definition["directory"]) / "result.json",
"frames": hardening_root / str(definition["directory"]) / "frames.jsonl",
}
for name, definition in RUNTIME_HARDENING_RUNS.items()
}
load_envelope_profile_path = (
repository / "config/perception/m48s-load-envelope-prefetch-v1.json"
)
load_envelope_paths = {
name: runtime
/ "load-envelope-prefetch-worker"
/ str(definition["run_id"])
/ "result.json"
for name, definition in LOAD_ENVELOPE_RUNS.items()
}
yolox_root = runtime / "yolox-all-coco-results" / YOLOX_ID yolox_root = runtime / "yolox-all-coco-results" / YOLOX_ID
yolox_manifest_path = yolox_root / "manifest.json" yolox_manifest_path = yolox_root / "manifest.json"
yolox_frames_path = yolox_root / "frames.jsonl" yolox_frames_path = yolox_root / "frames.jsonl"
@@ -137,6 +301,9 @@ def build_m48s_fixed_class_detector_lab(
dfine_path, dfine_path,
rf_detr_path, rf_detr_path,
profile_path, profile_path,
load_envelope_profile_path,
*load_envelope_paths.values(),
*(path for paths in hardening_paths.values() for path in paths.values()),
): ):
if path.is_symlink() or not path.is_file(): if path.is_symlink() or not path.is_file():
raise M48SFixedClassDetectorLabError( raise M48SFixedClassDetectorLabError(
@@ -150,6 +317,18 @@ def build_m48s_fixed_class_detector_lab(
reference_graph_worker = _read_object(reference_graph_worker_path) reference_graph_worker = _read_object(reference_graph_worker_path)
reference_graph_replay = _read_object(reference_graph_replay_path) reference_graph_replay = _read_object(reference_graph_replay_path)
reference_graph_replay_worker = _read_object(reference_graph_replay_worker_path) reference_graph_replay_worker = _read_object(reference_graph_replay_worker_path)
hardening_runs = {
name: _read_object(paths["result"]) for name, paths in hardening_paths.items()
}
load_envelope_profile = _read_object(load_envelope_profile_path)
load_envelope_runs = {
name: _read_object(path) for name, path in load_envelope_paths.items()
}
hardening_first_frames = {
name: _read_first_jsonl_object(paths["frames"])
for name, paths in hardening_paths.items()
if name in {"hardened", "prewarmed"}
}
yolox_manifest = _read_object(yolox_manifest_path) yolox_manifest = _read_object(yolox_manifest_path)
dfine = _read_object(dfine_path) dfine = _read_object(dfine_path)
rf_detr = _read_object(rf_detr_path) rf_detr = _read_object(rf_detr_path)
@@ -169,6 +348,13 @@ def build_m48s_fixed_class_detector_lab(
rf_detr=rf_detr, rf_detr=rf_detr,
profile=profile, profile=profile,
yolox_frames=yolox_frames, yolox_frames=yolox_frames,
hardening_runs=hardening_runs,
hardening_first_frames=hardening_first_frames,
hardening_paths=hardening_paths,
load_envelope_profile=load_envelope_profile,
load_envelope_profile_path=load_envelope_profile_path,
load_envelope_runs=load_envelope_runs,
load_envelope_paths=load_envelope_paths,
) )
source_paths = {frame_id: source_root / f"frame-{frame_id}.jpg" for frame_id in FRAME_IDS} source_paths = {frame_id: source_root / f"frame-{frame_id}.jpg" for frame_id in FRAME_IDS}
@@ -221,15 +407,26 @@ def build_m48s_fixed_class_detector_lab(
"reference_graph_document_sha256": sha256_path(reference_graph_path), "reference_graph_document_sha256": sha256_path(reference_graph_path),
"reference_graph_worker_sha256": sha256_path(reference_graph_worker_path), "reference_graph_worker_sha256": sha256_path(reference_graph_worker_path),
"reference_graph_replay_result_id": REFERENCE_GRAPH_REPLAY_ID, "reference_graph_replay_result_id": REFERENCE_GRAPH_REPLAY_ID,
"reference_graph_replay_document_sha256": sha256_path( "reference_graph_replay_document_sha256": sha256_path(reference_graph_replay_path),
reference_graph_replay_path "reference_graph_replay_worker_sha256": sha256_path(reference_graph_replay_worker_path),
), "reference_graph_replay_frames_sha256": sha256_path(reference_graph_replay_frames_path),
"reference_graph_replay_worker_sha256": sha256_path( "runtime_hardening": {
reference_graph_replay_worker_path name: {
), "result_sha256": sha256_path(paths["result"]),
"reference_graph_replay_frames_sha256": sha256_path( "frames_sha256": sha256_path(paths["frames"]),
reference_graph_replay_frames_path }
), for name, paths in hardening_paths.items()
},
"load_envelope": {
"profile_sha256": sha256_path(load_envelope_profile_path),
"runs": {
name: {
"result_sha256": sha256_path(load_envelope_paths[name]),
"frames_sha256": run["execution"]["frame_evidence"]["sha256"],
}
for name, run in load_envelope_runs.items()
},
},
"yolox_result_id": YOLOX_ID, "yolox_result_id": YOLOX_ID,
"yolox_document_sha256": sha256_path(yolox_manifest_path), "yolox_document_sha256": sha256_path(yolox_manifest_path),
}, },
@@ -238,7 +435,7 @@ def build_m48s_fixed_class_detector_lab(
} }
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest() identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
result_id = RESULT_PREFIX + identity_sha256 result_id = RESULT_PREFIX + identity_sha256
completed_utc_ns = reference_graph_replay_worker.get("completed_utc_ns") completed_utc_ns = load_envelope_runs["limit-15fps-a3"].get("completed_utc_ns")
if not isinstance(completed_utc_ns, int) or isinstance(completed_utc_ns, bool): if not isinstance(completed_utc_ns, int) or isinstance(completed_utc_ns, bool):
raise M48SFixedClassDetectorLabError( raise M48SFixedClassDetectorLabError(
"complete reference-graph completion time is unavailable" "complete reference-graph completion time is unavailable"
@@ -260,7 +457,11 @@ def build_m48s_fixed_class_detector_lab(
load=load, load=load,
reference_graph=reference_graph, reference_graph=reference_graph,
candidates=candidates, candidates=candidates,
hardening_runs=hardening_runs,
hardening_first_frames=hardening_first_frames,
load_envelope_runs=load_envelope_runs,
) )
envelope = cast(dict[str, Any], metrics["load_envelope"])
decision = { decision = {
"bounded_question_accepted": True, "bounded_question_accepted": True,
"selected_candidate": "rf-detr", "selected_candidate": "rf-detr",
@@ -268,6 +469,13 @@ def build_m48s_fixed_class_detector_lab(
"integrated_world_state_gate_evaluated": True, "integrated_world_state_gate_evaluated": True,
"integrated_world_state_gate_passed": True, "integrated_world_state_gate_passed": True,
"full_replay_visual_published": True, "full_replay_visual_published": True,
"load_envelope_evaluated": True,
"production_rate_repeatability_passed": envelope[
"production_rate_repeatability_passed"
],
"reserve_12_fps_passed": envelope["reserve_12_fps_passed"],
"limit_15_fps_passed": envelope["limit_15_fps_passed"],
"load_envelope_accepted": envelope["load_envelope_accepted"],
"detector_replacement_authorized": False, "detector_replacement_authorized": False,
"production_accepted": False, "production_accepted": False,
} }
@@ -283,8 +491,13 @@ def build_m48s_fixed_class_detector_lab(
"authority." "authority."
), ),
( (
"Eight source frames were superseded by the qualified latest-wins graph; their " "The visual timeline is the original qualified semantic replay; runtime-hardening "
"camera/LiDAR source evidence remains visible without invented world state." "metrics come from separately sealed, input-identical replay runs."
),
(
"Three complete repetitions at each of 10, 12 and 15 FPS passed their predeclared "
"integrity and operating-target gates after bounded decode prefetch was completed "
"before source admission."
), ),
] ]
@@ -370,6 +583,24 @@ def build_m48s_fixed_class_detector_lab(
reference_graph_replay_frames_path, reference_graph_replay_frames_path,
temporary / "reference-graph-replay-frames.jsonl", temporary / "reference-graph-replay-frames.jsonl",
) )
for name, paths in hardening_paths.items():
shutil.copyfile(paths["result"], temporary / f"runtime-hardening-{name}.json")
shutil.copyfile(
load_envelope_profile_path,
temporary / "runtime-load-envelope-profile.json",
)
for name, path in load_envelope_paths.items():
shutil.copyfile(path, temporary / f"runtime-load-{name}.json")
startup_evidence = {
"schema_version": RUNTIME_HARDENING_SCHEMA,
"source_frames_sha256": {
name: sha256_path(hardening_paths[name]["frames"])
for name in ("hardened", "prewarmed")
},
"first_delivered_frames": hardening_first_frames,
}
startup_path = temporary / "runtime-hardening-startup.json"
startup_path.write_bytes(canonical_json(startup_evidence) + b"\n")
report = { report = {
"schema_version": REPORT_SCHEMA, "schema_version": REPORT_SCHEMA,
"result_id": result_id, "result_id": result_id,
@@ -379,11 +610,16 @@ def build_m48s_fixed_class_detector_lab(
"execution": { "execution": {
"detector_load": load["execution"], "detector_load": load["execution"],
"complete_reference_graph": reference_graph["identity"]["evidence"]["execution"], "complete_reference_graph": reference_graph["identity"]["evidence"]["execution"],
"runtime_hardening": metrics["runtime_hardening"],
"load_envelope": metrics["load_envelope"],
}, },
"metrics": metrics, "metrics": metrics,
"acceptance": { "acceptance": {
"detector_load": load["checks"], "detector_load": load["checks"],
"complete_reference_graph": reference_graph["identity"]["evidence"]["checks"], "complete_reference_graph": reference_graph["identity"]["evidence"]["checks"],
"load_envelope": {
name: run["checks"] for name, run in load_envelope_runs.items()
},
}, },
"decision": decision, "decision": decision,
"limitations": limitations, "limitations": limitations,
@@ -450,6 +686,13 @@ def _validate_inputs(
rf_detr: dict[str, Any], rf_detr: dict[str, Any],
profile: dict[str, Any], profile: dict[str, Any],
yolox_frames: list[dict[str, Any]], yolox_frames: list[dict[str, Any]],
hardening_runs: dict[str, dict[str, Any]],
hardening_first_frames: dict[str, dict[str, Any]],
hardening_paths: dict[str, dict[str, Path]],
load_envelope_profile: dict[str, Any],
load_envelope_profile_path: Path,
load_envelope_runs: dict[str, dict[str, Any]],
load_envelope_paths: dict[str, Path],
) -> None: ) -> None:
decision = deployment.get("decision") decision = deployment.get("decision")
graph_identity = reference_graph.get("identity") graph_identity = reference_graph.get("identity")
@@ -460,8 +703,7 @@ def _validate_inputs(
replay_artifacts = reference_graph_replay.get("artifacts") replay_artifacts = reference_graph_replay.get("artifacts")
replay_frame_summary = ( replay_frame_summary = (
replay_identity.get("evidence", {}).get("frame_summary") replay_identity.get("evidence", {}).get("frame_summary")
if isinstance(replay_identity, dict) if isinstance(replay_identity, dict) and isinstance(replay_identity.get("evidence"), dict)
and isinstance(replay_identity.get("evidence"), dict)
else None else None
) )
if ( if (
@@ -539,6 +781,194 @@ def _validate_inputs(
authority = document.get("authority") authority = document.get("authority")
if authority is not None and authority != false_authority(): if authority is not None and authority != false_authority():
raise M48SFixedClassDetectorLabError("sealed M4.8S evidence gained authority") raise M48SFixedClassDetectorLabError("sealed M4.8S evidence gained authority")
for name, definition in RUNTIME_HARDENING_RUNS.items():
run = hardening_runs.get(name)
paths = hardening_paths.get(name)
if not isinstance(run, dict) or not isinstance(paths, dict):
raise M48SFixedClassDetectorLabError("runtime-hardening evidence is incomplete")
checks = run.get("checks")
execution = run.get("execution")
identity = run.get("identity")
frame_evidence = execution.get("frame_evidence") if isinstance(execution, dict) else None
if (
run.get("schema_version") != definition["schema_version"]
or run.get("completed") is not True
or run.get("integrated_runtime_gate_passed") is not True
or run.get("production_accepted") is not False
or run.get("authority") != false_authority()
or not isinstance(checks, dict)
or not checks
or not all(value is True for value in checks.values())
or not isinstance(identity, dict)
or identity.get("graph_id") != "reference-perception-graph/v2"
or identity.get("detector_provider_id")
!= "triton-rf-detr-large-coco-risk-fp16-shadow/v0"
or identity.get("worker_id") != "worker-006"
or not isinstance(execution, dict)
or execution.get("admitted_frames") != definition["admitted_count"]
or not isinstance(frame_evidence, dict)
or frame_evidence.get("row_count") != definition["frame_count"]
or frame_evidence.get("sha256") != definition["frames_sha256"]
or sha256_path(paths["result"]) != definition["result_sha256"]
or sha256_path(paths["frames"]) != definition["frames_sha256"]
):
raise M48SFixedClassDetectorLabError(
f"sealed runtime-hardening evidence changed: {name}"
)
for name in ("hardened", "prewarmed"):
frame = hardening_first_frames.get(name)
timing = frame.get("pipeline_timing") if isinstance(frame, dict) else None
detector = timing.get("detector") if isinstance(timing, dict) else None
if (
not isinstance(frame, dict)
or frame.get("schema_version") != "missioncore.m48s-reference-graph-frame-evidence/v1"
or not isinstance(timing, dict)
or timing.get("sequence") != 0
or not isinstance(timing.get("graph_admission_to_delivery_ns"), int)
or not isinstance(detector, dict)
or detector.get("sequence") != 0
or not isinstance(detector.get("total_duration_ns"), int)
):
raise M48SFixedClassDetectorLabError(
f"runtime-hardening startup evidence changed: {name}"
)
_validate_load_envelope(
profile=load_envelope_profile,
profile_path=load_envelope_profile_path,
runs=load_envelope_runs,
paths=load_envelope_paths,
)
def _validate_load_envelope(
*,
profile: dict[str, Any],
profile_path: Path,
runs: dict[str, dict[str, Any]],
paths: dict[str, Path],
) -> None:
scenarios = profile.get("scenarios")
if (
profile.get("schema_version") != "missioncore.m48s-load-envelope-profile/v2"
or profile.get("profile_id")
!= "m48s-rf-detr-reference-graph-load-envelope-prefetch/v1"
or sha256_path(profile_path) != LOAD_ENVELOPE_PROFILE_SHA256
or not isinstance(scenarios, list)
or len(scenarios) != 3
or profile.get("series_control", {}).get("repeat_count_per_rate") != 3
):
raise M48SFixedClassDetectorLabError("load-envelope profile identity changed")
scenarios_by_id = {
scenario.get("id"): scenario for scenario in scenarios if isinstance(scenario, dict)
}
expected_scenario_ids = {
str(definition["scenario_id"]) for definition in LOAD_ENVELOPE_RUNS.values()
}
if set(scenarios_by_id) != expected_scenario_ids:
raise M48SFixedClassDetectorLabError("load-envelope scenarios changed")
declared_run_ids = {
run_id
for scenario in scenarios_by_id.values()
for run_id in cast(list[object], scenario.get("run_ids", []))
}
expected_run_ids = {
str(definition["run_id"]) for definition in LOAD_ENVELOPE_RUNS.values()
}
if declared_run_ids != expected_run_ids:
raise M48SFixedClassDetectorLabError("load-envelope run series changed")
shared_inputs: dict[str, object] | None = None
for name, definition in LOAD_ENVELOPE_RUNS.items():
run = runs.get(name)
path = paths.get(name)
scenario = scenarios_by_id[str(definition["scenario_id"])]
if not isinstance(run, dict) or not isinstance(path, Path):
raise M48SFixedClassDetectorLabError("load-envelope evidence is incomplete")
source = run.get("source")
identity = run.get("identity")
execution = run.get("execution")
integrity_checks = run.get("integrity_checks")
thresholds = run.get("predeclared_thresholds")
frame_evidence = execution.get("frame_evidence") if isinstance(execution, dict) else None
terminal = execution.get("terminal_outcomes") if isinstance(execution, dict) else None
pipeline = run.get("metrics", {}).get("pipeline_timing")
decode_by_phase = (
pipeline.get("decode_duration_by_phase_ms") if isinstance(pipeline, dict) else None
)
pacing = (
pipeline.get("delivered_source_pacing_lateness_ms")
if isinstance(pipeline, dict)
else None
)
loops = execution.get("loops") if isinstance(execution, dict) else None
prefetch = (
loops[0].get("source_prefetch")
if isinstance(loops, list) and len(loops) == 1 and isinstance(loops[0], dict)
else None
)
if (
run.get("schema_version") != "missioncore.m48s-reference-graph-shadow-load/v5"
or run.get("completed") is not True
or run.get("production_accepted") is not False
or run.get("authority") != false_authority()
or run.get("evidence_integrity_gate_passed") is not True
or run.get("operating_target_gate_passed")
is not definition["operating_target_gate_passed"]
or not isinstance(integrity_checks, dict)
or not integrity_checks
or not all(value is True for value in integrity_checks.values())
or not isinstance(source, dict)
or source.get("source_id") != "RAVNOVES00"
or source.get("pacing_contract")
!= "wall-clock-scaled-source-timestamps-immutable/v1"
or source.get("requested_rate_hz") != definition["source_rate_hz"]
or not isinstance(identity, dict)
or identity.get("worker_id") != "worker-006"
or identity.get("graph_id") != "reference-perception-graph/v2"
or identity.get("detector_provider_id")
!= "triton-rf-detr-large-coco-risk-fp16-shadow/v0"
or identity.get("runtime_artifact_sha256") != LOAD_RUNTIME_ARTIFACT_SHA256
or identity.get("runner_sha256") != LOAD_RUNNER_SHA256
or not isinstance(execution, dict)
or execution.get("load_purpose") != definition["load_purpose"]
or execution.get("requested_source_rate_hz") != definition["source_rate_hz"]
or execution.get("source_timestamps_preserved") is not True
or execution.get("admitted_frames") != 4489
or execution.get("delivered_world_states") != definition["delivered"]
or not isinstance(terminal, dict)
or terminal.get("delivered") != definition["delivered"]
or terminal.get("superseded", 0) != definition["superseded"]
or sum(terminal.values()) != 4489
or not isinstance(frame_evidence, dict)
or frame_evidence.get("row_count") != definition["delivered"]
or frame_evidence.get("sha256") != definition["frames_sha256"]
or not isinstance(pipeline, dict)
or pipeline.get("additional_inference_passes") != 0
or not isinstance(decode_by_phase, dict)
or set(decode_by_phase) != {"preadmission", "hot-loop"}
or any(
not isinstance(decode_by_phase.get(phase), dict)
for phase in ("preadmission", "hot-loop")
)
or not isinstance(pacing, dict)
or not isinstance(prefetch, dict)
or prefetch.get("capacity_frames") != 64
or prefetch.get("ready_frames") != 64
or prefetch.get("buffered_frames") != 64
or not isinstance(prefetch.get("preparation_duration_ns"), int)
or thresholds != scenario.get("thresholds")
or definition["run_id"] not in scenario.get("run_ids", [])
or sha256_path(path) != definition["result_sha256"]
):
raise M48SFixedClassDetectorLabError(
f"sealed load-envelope evidence changed: {name}"
)
inputs = identity.get("inputs")
if not isinstance(inputs, dict):
raise M48SFixedClassDetectorLabError("load-envelope input identity is unavailable")
if shared_inputs is None:
shared_inputs = inputs
elif inputs != shared_inputs:
raise M48SFixedClassDetectorLabError("load-envelope runs changed graph inputs")
def _method( def _method(
@@ -674,6 +1104,9 @@ def _metrics(
load: dict[str, Any], load: dict[str, Any],
reference_graph: dict[str, Any], reference_graph: dict[str, Any],
candidates: list[dict[str, object]], candidates: list[dict[str, object]],
hardening_runs: dict[str, dict[str, Any]],
hardening_first_frames: dict[str, dict[str, Any]],
load_envelope_runs: dict[str, dict[str, Any]],
) -> dict[str, object]: ) -> dict[str, object]:
graph_evidence = reference_graph["identity"]["evidence"] graph_evidence = reference_graph["identity"]["evidence"]
graph_execution = graph_evidence["execution"] graph_execution = graph_evidence["execution"]
@@ -739,6 +1172,223 @@ def _metrics(
for key in ("failed", "stale", "rejected", "unavailable") for key in ("failed", "stale", "rejected", "unavailable")
), ),
}, },
"runtime_hardening": _runtime_hardening_metrics(
runs=hardening_runs,
first_frames=hardening_first_frames,
),
"load_envelope": _load_envelope_metrics(load_envelope_runs),
}
def _load_envelope_metrics(runs: dict[str, dict[str, Any]]) -> dict[str, object]:
scenarios: list[dict[str, object]] = []
scenario_order = ("production-10fps", "reserve-12fps", "limit-15fps")
for scenario_id in scenario_order:
definitions = sorted(
(
(name, definition)
for name, definition in LOAD_ENVELOPE_RUNS.items()
if definition["scenario_id"] == scenario_id
),
key=lambda item: int(item[1]["repetition"]),
)
repetitions: list[dict[str, object]] = []
queue_high_watermarks: dict[str, int] = {}
for name, definition in definitions:
run = runs[name]
execution = run["execution"]
metrics = run["metrics"]
completion = metrics["world_state_completion_age_ms"]
pipeline = metrics["pipeline_timing"]
decode = pipeline["decode_duration_by_phase_ms"]
pacing = pipeline["delivered_source_pacing_lateness_ms"]
terminal = execution["terminal_outcomes"]
prefetch = execution["loops"][0]["source_prefetch"]
for queue, depth in execution["queue_high_watermarks"].items():
queue_high_watermarks[queue] = max(
queue_high_watermarks.get(queue, 0), int(depth)
)
repetitions.append(
{
"run_id": definition["run_id"],
"repetition": definition["repetition"],
"delivered_world_states": execution["delivered_world_states"],
"superseded_frames": terminal.get("superseded", 0),
"delivery_ratio": execution["delivery_ratio"],
"effective_world_state_fps": execution["effective_world_state_fps"],
"world_state_completion_age_p95_ms": completion["p95"],
"hot_loop_decode_p95_ms": decode["hot-loop"]["p95"],
"hot_loop_decode_maximum_ms": decode["hot-loop"]["maximum"],
"preadmission_decode_maximum_ms": decode["preadmission"]["maximum"],
"source_pacing_lateness_maximum_ms": pacing["maximum"],
"source_prefetch_preparation_ms": (
prefetch["preparation_duration_ns"] / 1_000_000
),
"operating_target_gate_passed": run["operating_target_gate_passed"],
"result_sha256": definition["result_sha256"],
"frame_evidence_sha256": execution["frame_evidence"]["sha256"],
}
)
first_name = definitions[0][0]
first_run = runs[first_name]
run_values = [runs[name] for name, _definition in definitions]
executions = [run["execution"] for run in run_values]
run_metrics = [run["metrics"] for run in run_values]
pipelines = [metrics["pipeline_timing"] for metrics in run_metrics]
completions = [metrics["world_state_completion_age_ms"] for metrics in run_metrics]
gpus = [metrics["gpu"] for metrics in run_metrics]
all_integrity = all(run["evidence_integrity_gate_passed"] for run in run_values)
all_targets = all(run["operating_target_gate_passed"] for run in run_values)
scenarios.append(
{
"id": scenario_id,
"load_purpose": first_run["execution"]["load_purpose"],
"requested_source_rate_hz": first_run["execution"][
"requested_source_rate_hz"
],
"repeat_count": len(repetitions),
"all_repetitions_passed": all_integrity and all_targets,
"source_frames_admitted": min(
execution["admitted_frames"] for execution in executions
),
"delivered_world_states": min(
execution["delivered_world_states"] for execution in executions
),
"superseded_frames": max(
execution["terminal_outcomes"].get("superseded", 0)
for execution in executions
),
"delivery_ratio": min(execution["delivery_ratio"] for execution in executions),
"effective_world_state_fps": min(
execution["effective_world_state_fps"] for execution in executions
),
"world_state_completion_age_p95_ms": max(
completion["p95"] for completion in completions
),
"world_state_completion_age_p99_ms": max(
completion["p99"] for completion in completions
),
"world_state_completion_age_maximum_ms": max(
completion["maximum"] for completion in completions
),
"hot_loop_decode_p95_ms": max(
pipeline["decode_duration_by_phase_ms"]["hot-loop"]["p95"]
for pipeline in pipelines
),
"hot_loop_decode_maximum_ms": max(
pipeline["decode_duration_by_phase_ms"]["hot-loop"]["maximum"]
for pipeline in pipelines
),
"preadmission_decode_maximum_ms": max(
pipeline["decode_duration_by_phase_ms"]["preadmission"]["maximum"]
for pipeline in pipelines
),
"source_prefetch_preparation_maximum_ms": max(
repetition["source_prefetch_preparation_ms"]
for repetition in repetitions
),
"source_pacing_lateness_p95_ms": max(
pipeline["delivered_source_pacing_lateness_ms"]["p95"]
for pipeline in pipelines
),
"source_pacing_lateness_maximum_ms": max(
pipeline["delivered_source_pacing_lateness_ms"]["maximum"]
for pipeline in pipelines
),
"detector_p95_ms": max(
pipeline["detector_ms"]["total"]["p95"] for pipeline in pipelines
),
"detector_maximum_ms": max(
pipeline["detector_ms"]["total"]["maximum"]
for pipeline in pipelines
),
"gpu_utilization_mean_percent": max(
gpu["gpu_utilization_percent"]["mean"] for gpu in gpus
),
"gpu_utilization_maximum_percent": max(
gpu["gpu_utilization_percent"]["maximum"] for gpu in gpus
),
"gpu_memory_maximum_mib": max(
gpu["gpu_memory_used_mib"]["maximum"] for gpu in gpus
),
"process_peak_rss_mib": max(
metrics["process_peak_rss_after_mib"] for metrics in run_metrics
),
"queue_high_watermarks": queue_high_watermarks,
"queue_capacity": 2,
"additional_inference_passes": max(
pipeline["additional_inference_passes"] for pipeline in pipelines
),
"integrity_gate_passed": all_integrity,
"operating_target_gate_passed": all_targets,
"thresholds": first_run["predeclared_thresholds"],
"repetitions": repetitions,
}
)
production_passed = cast(bool, scenarios[0]["all_repetitions_passed"])
reserve_passed = cast(bool, scenarios[1]["all_repetitions_passed"])
limit_passed = cast(bool, scenarios[2]["all_repetitions_passed"])
return {
"schema_version": LOAD_ENVELOPE_SCHEMA,
"repeat_count_per_rate": 3,
"production_rate_repeatability_passed": production_passed,
"reserve_12_fps_passed": reserve_passed,
"limit_15_fps_passed": limit_passed,
"load_envelope_accepted": production_passed and reserve_passed,
"compute_capacity_at_least_fps": (
15.0 if limit_passed else 12.0 if reserve_passed else 10.0
),
"bottleneck_interpretation": (
"cold-video-decode-isolated-before-admission-no-steady-gpu-saturation"
),
"scenarios": scenarios,
}
def _runtime_hardening_metrics(
*,
runs: dict[str, dict[str, Any]],
first_frames: dict[str, dict[str, Any]],
) -> dict[str, object]:
def full_run(name: str) -> dict[str, object]:
run = runs[name]
execution = run["execution"]
completion = run["metrics"]["world_state_completion_age_ms"]
pipeline = run["metrics"]["pipeline_timing"]
terminal = execution["terminal_outcomes"]
return {
"source_frames_admitted": execution["admitted_frames"],
"delivered_world_states": execution["delivered_world_states"],
"superseded_frames": terminal.get("superseded", 0),
"effective_world_state_fps": execution["effective_world_state_fps"],
"world_state_completion_age_p95_ms": completion["p95"],
"world_state_completion_age_p99_ms": completion["p99"],
"world_state_completion_age_maximum_ms": completion["maximum"],
"rolling_maximum_ms": pipeline["provider_ms"]["rolling"]["maximum"],
"geometry_maximum_ms": pipeline["provider_ms"]["geometry"]["maximum"],
"additional_inference_passes": pipeline["additional_inference_passes"],
}
def startup_frame(name: str) -> dict[str, float]:
timing = first_frames[name]["pipeline_timing"]
return {
"detector_ms": timing["detector"]["total_duration_ns"] / 1_000_000,
"world_state_ms": timing["graph_admission_to_delivery_ns"] / 1_000_000,
}
prewarmed_loop = runs["prewarmed"]["execution"]["loops"][0]
warmup = prewarmed_loop["detector_warmup"]
return {
"schema_version": RUNTIME_HARDENING_SCHEMA,
"baseline": full_run("baseline"),
"hardened": full_run("hardened"),
"startup": {
"baseline": startup_frame("hardened"),
"prewarmed": startup_frame("prewarmed"),
"prewarm_duration_ms": warmup["total_duration_ns"] / 1_000_000,
"prewarm_inference_passes": warmup["inference_passes"],
"validation_frames": runs["prewarmed"]["execution"]["admitted_frames"],
},
} }
@@ -843,6 +1493,17 @@ def _artifact_manifest(root: Path) -> list[dict[str, object]]:
media_type = "application/x-ndjson" media_type = "application/x-ndjson"
schema_version = "missioncore.m48s-reference-graph-frame-evidence/v0" schema_version = "missioncore.m48s-reference-graph-frame-evidence/v0"
role = "visual-evidence-full-replay-world-state" role = "visual-evidence-full-replay-world-state"
elif relative.startswith("runtime-hardening-"):
schema_version = (
RUNTIME_HARDENING_SCHEMA if relative == "runtime-hardening-startup.json" else None
)
role = "upstream-runtime-hardening-evidence"
elif relative == "runtime-load-envelope-profile.json":
schema_version = "missioncore.m48s-load-envelope-profile/v2"
role = "predeclared-load-envelope-profile"
elif relative.startswith("runtime-load-"):
schema_version = "missioncore.m48s-reference-graph-shadow-load/v5"
role = "upstream-load-envelope-evidence"
artifacts.append( artifacts.append(
{ {
"role": role, "role": role,
@@ -866,6 +1527,18 @@ def _read_object(path: Path) -> dict[str, Any]:
return value return value
def _read_first_jsonl_object(path: Path) -> dict[str, Any]:
try:
with path.open("r", encoding="utf-8") as stream:
line = stream.readline()
value = json.loads(line)
except (OSError, json.JSONDecodeError) as exc:
raise M48SFixedClassDetectorLabError(f"invalid first-frame evidence: {path.name}") from exc
if not isinstance(value, dict):
raise M48SFixedClassDetectorLabError(f"first-frame evidence must be an object: {path.name}")
return value
def _read_jsonl(path: Path) -> list[dict[str, Any]]: def _read_jsonl(path: Path) -> list[dict[str, Any]]:
try: try:
rows = [json.loads(line) for line in path.read_text("utf-8").splitlines() if line] rows = [json.loads(line) for line in path.read_text("utf-8").splitlines() if line]
+132
View File
@@ -0,0 +1,132 @@
"""Versioned launchd declaration for the canonical local Mission Core service."""
from __future__ import annotations
import hashlib
import plistlib
from dataclasses import dataclass
from pathlib import Path
from typing import Final
MISSION_CORE_LAUNCH_AGENT_LABEL: Final = "com.nodedc.mission-core.local"
MISSION_CORE_LAUNCH_AGENT_SCHEMA: Final = "missioncore.local-launch-agent-plan/v1"
class MissionCoreLaunchAgentError(RuntimeError):
"""The local launch agent cannot be planned without weakening its boundary."""
@dataclass(frozen=True, slots=True)
class MissionCoreLaunchAgentPlan:
agent_path: Path
current_sha256: str
desired_sha256: str
current_program_arguments: tuple[str, ...]
desired_program_arguments: tuple[str, ...]
desired_payload: bytes
def to_dict(self) -> dict[str, object]:
return {
"schema_version": MISSION_CORE_LAUNCH_AGENT_SCHEMA,
"label": MISSION_CORE_LAUNCH_AGENT_LABEL,
"agent_path": str(self.agent_path),
"current_sha256": self.current_sha256,
"desired_sha256": self.desired_sha256,
"current_program_arguments": list(self.current_program_arguments),
"desired_program_arguments": list(self.desired_program_arguments),
"changes": {
"dependency_sync_disabled": "--no-sync"
in self.desired_program_arguments,
"self_health_watchdog": True,
"bounded_launchd_exit_timeout_seconds": 20,
"keep_alive": True,
"process_group_owned": True,
},
}
def plan_mission_core_launch_agent(
*,
repository_root: Path,
agent_path: Path,
) -> MissionCoreLaunchAgentPlan:
repository = repository_root.expanduser().resolve(strict=True)
path = agent_path.expanduser().absolute()
current_payload = _read_private_regular_file(path)
try:
current = plistlib.loads(current_payload)
except plistlib.InvalidFileException as exc:
raise MissionCoreLaunchAgentError("current Mission Core launch agent is invalid") from exc
if not isinstance(current, dict) or current.get("Label") != MISSION_CORE_LAUNCH_AGENT_LABEL:
raise MissionCoreLaunchAgentError("current launch agent identity changed")
current_arguments = _program_arguments(current)
current_working_directory = current.get("WorkingDirectory")
if current_working_directory != str(repository):
raise MissionCoreLaunchAgentError("current launch agent targets another repository")
environment = current.get("EnvironmentVariables")
if not isinstance(environment, dict) or any(
not isinstance(key, str) or not isinstance(value, str)
for key, value in environment.items()
):
raise MissionCoreLaunchAgentError("current launch agent environment is invalid")
# A LaunchAgent started directly from this repository's venv is denied
# access to ``.venv/pyvenv.cfg`` by macOS privacy controls because the
# checkout is below Downloads. The Homebrew uv launcher is already the
# accepted local execution boundary. ``--no-sync`` keeps launch startup
# deterministic and prevents dependency mutation during recovery.
uv_entrypoint = Path(current_arguments[0])
if (
not uv_entrypoint.is_absolute()
or uv_entrypoint.name != "uv"
or not uv_entrypoint.exists()
):
raise MissionCoreLaunchAgentError("Mission Core uv entrypoint is unavailable")
desired_environment = dict(environment)
desired_environment["MISSIONCORE_SERVICE_WATCHDOG"] = "1"
log_path = repository / ".runtime/mission-core/k1link-serve-launchd.log"
desired: dict[str, object] = {
"Label": MISSION_CORE_LAUNCH_AGENT_LABEL,
"ProgramArguments": [
str(uv_entrypoint),
"run",
"--no-sync",
"k1link",
"serve",
],
"WorkingDirectory": str(repository),
"EnvironmentVariables": desired_environment,
"KeepAlive": True,
"RunAtLoad": True,
"AbandonProcessGroup": False,
"ProcessType": "Background",
"ThrottleInterval": 5,
"ExitTimeOut": 20,
"StandardOutPath": str(log_path),
"StandardErrorPath": str(log_path),
}
desired_payload = plistlib.dumps(desired, fmt=plistlib.FMT_XML, sort_keys=True)
return MissionCoreLaunchAgentPlan(
agent_path=path,
current_sha256=_sha256(current_payload),
desired_sha256=_sha256(desired_payload),
current_program_arguments=current_arguments,
desired_program_arguments=tuple(desired["ProgramArguments"]),
desired_payload=desired_payload,
)
def _program_arguments(document: dict[str, object]) -> tuple[str, ...]:
value = document.get("ProgramArguments")
if not isinstance(value, list) or not value or any(not isinstance(item, str) for item in value):
raise MissionCoreLaunchAgentError("launch agent program arguments are invalid")
return tuple(value)
def _read_private_regular_file(path: Path) -> bytes:
if path.is_symlink() or not path.is_file():
raise MissionCoreLaunchAgentError("Mission Core launch agent is unavailable")
return path.read_bytes()
def _sha256(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
@@ -5,7 +5,7 @@ from __future__ import annotations
import hashlib import hashlib
import json import json
from collections.abc import Callable, Iterator from collections.abc import Callable, Iterator
from dataclasses import dataclass from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from threading import Event from threading import Event
@@ -32,10 +32,13 @@ from .providers import (
) )
from .recorded_source import ( from .recorded_source import (
DecodedRecordedSource, DecodedRecordedSource,
DecodePrefetchSnapshot,
DecodeTimingObserver, DecodeTimingObserver,
PrefetchedRecordedImageDecoder,
PyAvRecordedImageDecoder, PyAvRecordedImageDecoder,
RecordedRavnoves00Source, RecordedRavnoves00Source,
ReplayPacing, ReplayPacing,
SourcePacingObserver,
) )
from .reference_graph_runtime import ReferenceGraphRuntimePaths from .reference_graph_runtime import ReferenceGraphRuntimePaths
from .rf_detr_object_detector import ( from .rf_detr_object_detector import (
@@ -64,6 +67,8 @@ class M48sReferenceGraphRuntime:
graph: ReferencePerceptionGraphV2 graph: ReferencePerceptionGraphV2
inference_backend: TritonRfDetrHttpInferenceBackend inference_backend: TritonRfDetrHttpInferenceBackend
source_prefetch: PrefetchedRecordedImageDecoder
_preparation_stop_event: Event = field(default_factory=Event)
def warm_up_detector(self) -> DetectorWarmupSnapshot: def warm_up_detector(self) -> DetectorWarmupSnapshot:
detector = self.graph.detector detector = self.graph.detector
@@ -71,8 +76,18 @@ class M48sReferenceGraphRuntime:
raise M48sReferenceGraphRuntimeError("RF-DETR runtime detector changed before warmup") raise M48sReferenceGraphRuntimeError("RF-DETR runtime detector changed before warmup")
return detector.warm_up() return detector.warm_up()
def prepare_source(self) -> DecodePrefetchSnapshot:
return self.source_prefetch.prepare(self._preparation_stop_event)
def mark_source_admission_started(self) -> None:
self.source_prefetch.mark_admission_started()
def close(self) -> None: def close(self) -> None:
self.inference_backend.close() self._preparation_stop_event.set()
try:
self.source_prefetch.close()
finally:
self.inference_backend.close()
def __enter__(self) -> M48sReferenceGraphRuntime: def __enter__(self) -> M48sReferenceGraphRuntime:
return self return self
@@ -90,8 +105,12 @@ def build_m48s_reference_graph_runtime(
delivery_observer: Callable[[DeliveredFrame, int], None] | None = None, delivery_observer: Callable[[DeliveredFrame, int], None] | None = None,
delivery_evidence_observer: DeliveryEvidenceObserver | None = None, delivery_evidence_observer: DeliveryEvidenceObserver | None = None,
decode_timing_observer: DecodeTimingObserver | None = None, decode_timing_observer: DecodeTimingObserver | None = None,
source_pacing_observer: SourcePacingObserver | None = None,
detector_timing_observer: DetectorTimingObserver | None = None, detector_timing_observer: DetectorTimingObserver | None = None,
maximum_frames: int | None = None, maximum_frames: int | None = None,
source_rate_hz: float | None = None,
source_prefetch_capacity_frames: int = 64,
source_prefetch_ready_frames: int = 64,
) -> M48sReferenceGraphRuntime: ) -> M48sReferenceGraphRuntime:
"""Instantiate the complete graph with only its detector pin replaced.""" """Instantiate the complete graph with only its detector pin replaced."""
@@ -116,6 +135,12 @@ def build_m48s_reference_graph_runtime(
if maximum_frames is not None and maximum_frames < 1: if maximum_frames is not None and maximum_frames < 1:
raise M48sReferenceGraphRuntimeError("maximum frame count must be positive") raise M48sReferenceGraphRuntimeError("maximum frame count must be positive")
prefetch = PrefetchedRecordedImageDecoder(
PyAvRecordedImageDecoder(paths.video),
capacity_frames=source_prefetch_capacity_frames,
ready_frames=source_prefetch_ready_frames,
timing_observer=decode_timing_observer,
)
source: SourceProvider = DecodedRecordedSource( source: SourceProvider = DecodedRecordedSource(
source=RecordedRavnoves00Source( source=RecordedRavnoves00Source(
camera_index_path=paths.camera_index, camera_index_path=paths.camera_index,
@@ -125,9 +150,14 @@ def build_m48s_reference_graph_runtime(
if run_mode is GraphRunMode.SOURCE_PACED_LATEST_WINS if run_mode is GraphRunMode.SOURCE_PACED_LATEST_WINS
else ReplayPacing.UNCAPPED else ReplayPacing.UNCAPPED
), ),
target_rate_hz=(
source_rate_hz
if run_mode is GraphRunMode.SOURCE_PACED_LATEST_WINS
else None
),
pacing_observer=source_pacing_observer,
), ),
decoder=PyAvRecordedImageDecoder(paths.video), decoder=prefetch,
timing_observer=decode_timing_observer,
) )
if maximum_frames is not None: if maximum_frames is not None:
source = _LimitedSource(source, maximum_frames) source = _LimitedSource(source, maximum_frames)
@@ -169,9 +199,16 @@ def build_m48s_reference_graph_runtime(
delivery_evidence_observer=delivery_evidence_observer, delivery_evidence_observer=delivery_evidence_observer,
) )
except Exception: except Exception:
backend.close() try:
prefetch.close()
finally:
backend.close()
raise raise
return M48sReferenceGraphRuntime(graph=graph, inference_backend=backend) return M48sReferenceGraphRuntime(
graph=graph,
inference_backend=backend,
source_prefetch=prefetch,
)
class _LimitedSource: class _LimitedSource:
+260 -3
View File
@@ -10,7 +10,8 @@ from collections.abc import Callable, Iterator
from dataclasses import dataclass, replace from dataclasses import dataclass, replace
from enum import StrEnum from enum import StrEnum
from pathlib import Path from pathlib import Path
from threading import Event from queue import Empty, Full, Queue
from threading import Event, Lock, Thread
from typing import Any, Final, Protocol, cast from typing import Any, Final, Protocol, cast
import numpy as np import numpy as np
@@ -53,6 +54,11 @@ class ReplayPacing(StrEnum):
UNCAPPED = "uncapped" UNCAPPED = "uncapped"
class DecodePhase(StrEnum):
PREADMISSION = "preadmission"
HOT_LOOP = "hot-loop"
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
class RecordedFrameReference: class RecordedFrameReference:
"""Opaque reference passed to a provider without decoding sensor data.""" """Opaque reference passed to a provider without decoding sensor data."""
@@ -89,6 +95,7 @@ WaitFunction = Callable[[Event, float], bool]
class DecodedFrameTiming: class DecodedFrameTiming:
sequence: int sequence: int
duration_ns: int duration_ns: int
phase: DecodePhase = DecodePhase.HOT_LOOP
def __post_init__(self) -> None: def __post_init__(self) -> None:
if self.sequence < 0 or self.duration_ns < 0: if self.sequence < 0 or self.duration_ns < 0:
@@ -98,8 +105,49 @@ class DecodedFrameTiming:
DecodeTimingObserver = Callable[[DecodedFrameTiming], None] DecodeTimingObserver = Callable[[DecodedFrameTiming], None]
@dataclass(frozen=True, slots=True)
class SourcePacingTiming:
sequence: int
scheduled_monotonic_ns: int
emitted_monotonic_ns: int
lateness_ns: int
def __post_init__(self) -> None:
if (
self.sequence < 0
or self.scheduled_monotonic_ns < 0
or self.emitted_monotonic_ns < 0
or self.lateness_ns < 0
):
raise RecordedSourceError("source pacing timing must be nonnegative")
SourcePacingObserver = Callable[[SourcePacingTiming], None]
@dataclass(frozen=True, slots=True)
class DecodePrefetchSnapshot:
capacity_frames: int
ready_frames: int
buffered_frames: int
preparation_duration_ns: int
producer_alive: bool
@dataclass(frozen=True, slots=True)
class _DecodeFailure:
error: BaseException
_DECODE_END: Final = object()
class RecordedRavnoves00Source: class RecordedRavnoves00Source:
"""Emit the admitted synchronized source timeline at 1.0x or uncapped speed.""" """Emit the admitted timeline at its recorded or an explicit replay rate.
A target rate changes wall-clock pacing only. Immutable sensor timestamps stay
untouched so accelerated load measurements cannot silently change scene motion.
"""
provider_id: str = RECORDED_SOURCE_PROVIDER_ID provider_id: str = RECORDED_SOURCE_PROVIDER_ID
@@ -111,16 +159,26 @@ class RecordedRavnoves00Source:
pacing: ReplayPacing = ReplayPacing.UNCAPPED, pacing: ReplayPacing = ReplayPacing.UNCAPPED,
expected_frame_count: int = DEFAULT_FRAME_COUNT, expected_frame_count: int = DEFAULT_FRAME_COUNT,
expected_source_pack_sha256: str | None = RECORDED_SOURCE_PACK_SHA256, expected_source_pack_sha256: str | None = RECORDED_SOURCE_PACK_SHA256,
target_rate_hz: float | None = None,
pacing_observer: SourcePacingObserver | None = None,
clock_ns: Callable[[], int] = time.monotonic_ns, clock_ns: Callable[[], int] = time.monotonic_ns,
wait: WaitFunction | None = None, wait: WaitFunction | None = None,
) -> None: ) -> None:
if expected_frame_count < 1: if expected_frame_count < 1:
raise RecordedSourceError("expected frame count must be positive") raise RecordedSourceError("expected frame count must be positive")
if target_rate_hz is not None and (
not np.isfinite(target_rate_hz) or target_rate_hz <= 0
):
raise RecordedSourceError("target replay rate must be positive and finite")
if target_rate_hz is not None and pacing is not ReplayPacing.ONE_X:
raise RecordedSourceError("target replay rate requires paced replay")
self.camera_index_path = camera_index_path.resolve() self.camera_index_path = camera_index_path.resolve()
self.source_pack_path = source_pack_path.resolve() self.source_pack_path = source_pack_path.resolve()
self.pacing = pacing self.pacing = pacing
self.expected_frame_count = expected_frame_count self.expected_frame_count = expected_frame_count
self.expected_source_pack_sha256 = expected_source_pack_sha256 self.expected_source_pack_sha256 = expected_source_pack_sha256
self.target_rate_hz = target_rate_hz
self.pacing_observer = pacing_observer
self._clock_ns = clock_ns self._clock_ns = clock_ns
self._wait = wait or _event_wait self._wait = wait or _event_wait
@@ -130,6 +188,7 @@ class RecordedRavnoves00Source:
repository_root: Path, repository_root: Path,
*, *,
pacing: ReplayPacing = ReplayPacing.UNCAPPED, pacing: ReplayPacing = ReplayPacing.UNCAPPED,
target_rate_hz: float | None = None,
) -> RecordedRavnoves00Source: ) -> RecordedRavnoves00Source:
root = repository_root.resolve() root = repository_root.resolve()
camera_index = ( camera_index = (
@@ -148,6 +207,7 @@ class RecordedRavnoves00Source:
camera_index_path=camera_index, camera_index_path=camera_index,
source_pack_path=source_pack, source_pack_path=source_pack,
pacing=pacing, pacing=pacing,
target_rate_hz=target_rate_hz,
) )
def packets(self, stop_event: Event) -> Iterator[SourcePacket]: def packets(self, stop_event: Event) -> Iterator[SourcePacket]:
@@ -163,6 +223,7 @@ class RecordedRavnoves00Source:
started_ns = int(self._clock_ns()) started_ns = int(self._clock_ns())
source_origin_ns = _source_time_ns(timeline_rows[0]) source_origin_ns = _source_time_ns(timeline_rows[0])
pacing_scale = _pacing_scale(timeline_rows, self.target_rate_hz)
for frame_index, (camera, timeline) in enumerate( for frame_index, (camera, timeline) in enumerate(
zip(camera_rows, timeline_rows, strict=True) zip(camera_rows, timeline_rows, strict=True)
): ):
@@ -170,9 +231,20 @@ class RecordedRavnoves00Source:
return return
packet = _packet(frame_index, camera, timeline) packet = _packet(frame_index, camera, timeline)
if self.pacing is ReplayPacing.ONE_X: if self.pacing is ReplayPacing.ONE_X:
target_ns = started_ns + packet.envelope.timestamps.source_ns - source_origin_ns source_elapsed_ns = packet.envelope.timestamps.source_ns - source_origin_ns
target_ns = started_ns + round(source_elapsed_ns * pacing_scale)
if not self._pace_until(stop_event, target_ns): if not self._pace_until(stop_event, target_ns):
return return
emitted_ns = int(self._clock_ns())
if self.pacing_observer is not None:
self.pacing_observer(
SourcePacingTiming(
sequence=packet.envelope.sequence,
scheduled_monotonic_ns=target_ns,
emitted_monotonic_ns=emitted_ns,
lateness_ns=max(0, emitted_ns - target_ns),
)
)
yield packet yield packet
def _pace_until(self, stop_event: Event, target_ns: int) -> bool: def _pace_until(self, stop_event: Event, target_ns: int) -> bool:
@@ -279,6 +351,176 @@ class PyAvRecordedImageDecoder:
container.close() container.close()
class PrefetchedRecordedImageDecoder:
"""Decode into a bounded queue before source admission and during replay.
The queue absorbs ordinary storage/codec jitter without retaining the whole
recording in RAM. Decode work remains single-pass and source order remains
exact. The explicit phase boundary keeps cold codec initialization out of
hot-loop latency attribution.
"""
def __init__(
self,
decoder: RecordedImageDecoder,
*,
capacity_frames: int = 64,
ready_frames: int = 64,
timing_observer: DecodeTimingObserver | None = None,
clock_ns: Callable[[], int] = time.perf_counter_ns,
) -> None:
if capacity_frames < 1:
raise RecordedSourceError("decode prefetch capacity must be positive")
if ready_frames < 1 or ready_frames > capacity_frames:
raise RecordedSourceError("decode prefetch readiness must fit its capacity")
self.decoder = decoder
self.capacity_frames = capacity_frames
self.ready_frames = ready_frames
self.timing_observer = timing_observer
self._clock_ns = clock_ns
self._queue: Queue[NDArray[np.uint8] | _DecodeFailure | object] = Queue(
capacity_frames
)
self._stop = Event()
self._ready = Event()
self._guard = Lock()
self._thread: Thread | None = None
self._phase = DecodePhase.PREADMISSION
self._produced = 0
self._started_ns: int | None = None
self._ready_ns: int | None = None
self._closed = False
def prepare(self, stop_event: Event) -> DecodePrefetchSnapshot:
with self._guard:
if self._closed:
raise RecordedSourceError("decode prefetch is closed")
if self._thread is None:
self._started_ns = int(self._clock_ns())
self._thread = Thread(
target=self._produce,
name="m48s-recorded-decode-prefetch",
daemon=True,
)
self._thread.start()
while not self._ready.wait(0.05):
if stop_event.is_set():
raise RecordedSourceError("decode prefetch preparation was cancelled")
with self._guard:
started_ns = self._started_ns
ready_ns = self._ready_ns
thread = self._thread
produced = self._produced
if started_ns is None or ready_ns is None or thread is None:
raise RecordedSourceError("decode prefetch readiness is incomplete")
first = self._peek()
if isinstance(first, _DecodeFailure):
raise RecordedSourceError("recorded camera prefetch failed") from first.error
return DecodePrefetchSnapshot(
capacity_frames=self.capacity_frames,
ready_frames=self.ready_frames,
buffered_frames=min(produced, self.capacity_frames),
preparation_duration_ns=max(0, ready_ns - started_ns),
producer_alive=thread.is_alive(),
)
def mark_admission_started(self) -> None:
with self._guard:
if not self._ready.is_set():
raise RecordedSourceError("source admission started before decode prefetch")
self._phase = DecodePhase.HOT_LOOP
def frames(self, stop_event: Event) -> Iterator[NDArray[np.uint8]]:
self.prepare(stop_event)
while not stop_event.is_set() and not self._stop.is_set():
try:
item = self._queue.get(timeout=0.05)
except Empty:
continue
try:
if item is _DECODE_END:
return
if isinstance(item, _DecodeFailure):
raise RecordedSourceError("recorded camera decode failed") from item.error
yield cast(NDArray[np.uint8], item)
finally:
self._queue.task_done()
def close(self, *, timeout_seconds: float = 5.0) -> None:
with self._guard:
if self._closed:
return
self._closed = True
self._stop.set()
thread = self._thread
if thread is not None:
thread.join(timeout=max(0.0, timeout_seconds))
if thread.is_alive():
raise RecordedSourceError("decode prefetch worker did not stop")
def _produce(self) -> None:
sequence = 0
images = self.decoder.frames(self._stop)
try:
while not self._stop.is_set():
while self._queue.full() and not self._stop.wait(0.01):
pass
if self._stop.is_set():
return
started_ns = int(self._clock_ns())
try:
image = next(images)
except StopIteration:
self._signal_ready()
self._put(_DECODE_END)
return
completed_ns = int(self._clock_ns())
with self._guard:
phase = self._phase
if self.timing_observer is not None:
self.timing_observer(
DecodedFrameTiming(
sequence=sequence,
duration_ns=max(0, completed_ns - started_ns),
phase=phase,
)
)
if not self._put(np.asarray(image, dtype=np.uint8)):
return
sequence += 1
with self._guard:
self._produced = sequence
ready = sequence >= self.ready_frames
if ready:
self._signal_ready()
except BaseException as exc:
self._put(_DecodeFailure(exc))
self._signal_ready()
finally:
close = getattr(images, "close", None)
if callable(close):
close()
def _put(self, item: NDArray[np.uint8] | _DecodeFailure | object) -> bool:
while not self._stop.is_set():
try:
self._queue.put(item, timeout=0.05)
return True
except Full:
continue
return False
def _signal_ready(self) -> None:
with self._guard:
if self._ready_ns is None:
self._ready_ns = int(self._clock_ns())
self._ready.set()
def _peek(self) -> NDArray[np.uint8] | _DecodeFailure | object | None:
with self._queue.mutex:
return self._queue.queue[0] if self._queue.queue else None
def _packet( def _packet(
frame_index: int, frame_index: int,
camera: dict[str, object], camera: dict[str, object],
@@ -349,6 +591,21 @@ def _source_time_ns(document: _SourceTimelineRow) -> int:
return round(value * 1_000_000_000) return round(value * 1_000_000_000)
def _pacing_scale(
timeline: tuple[_SourceTimelineRow, ...],
target_rate_hz: float | None,
) -> float:
if target_rate_hz is None:
return 1.0
if len(timeline) < 2:
raise RecordedSourceError("target-rate replay requires at least two frames")
duration_seconds = timeline[-1].session_seconds - timeline[0].session_seconds
if not np.isfinite(duration_seconds) or duration_seconds <= 0:
raise RecordedSourceError("source timeline duration is invalid")
recorded_rate_hz = (len(timeline) - 1) / duration_seconds
return recorded_rate_hz / target_rate_hz
def _integer(document: dict[str, object], key: str) -> int: def _integer(document: dict[str, object], key: str) -> int:
value = document.get(key) value = document.get(key)
if not isinstance(value, int) or isinstance(value, bool) or value < 0: if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+272
View File
@@ -0,0 +1,272 @@
"""Fail-closed health watchdog for the canonical Mission Core service process."""
from __future__ import annotations
import http.client
import json
import os
import signal
import stat
import time
from collections.abc import Callable, Mapping
from contextlib import suppress
from dataclasses import dataclass
from pathlib import Path
from threading import Event, Lock, Thread
from typing import Final
WATCHDOG_SCHEMA: Final = "missioncore.local-service-watchdog/v1"
WATCHDOG_ENV: Final = "MISSIONCORE_SERVICE_WATCHDOG"
WATCHDOG_ENABLED_VALUE: Final = "1"
MISSION_CORE_SERVICE_ID: Final = "mission-core-control-plane"
DEFAULT_JOURNAL_MAX_BYTES: Final = 4 * 1024 * 1024
class MissionCoreWatchdogError(RuntimeError):
"""The watchdog cannot establish a trustworthy local safety boundary."""
@dataclass(frozen=True, slots=True)
class MissionCoreWatchdogPolicy:
startup_grace_seconds: float = 45.0
probe_interval_seconds: float = 2.0
probe_timeout_seconds: float = 1.0
consecutive_failure_limit: int = 3
graceful_shutdown_seconds: float = 12.0
def __post_init__(self) -> None:
if (
self.startup_grace_seconds <= 0
or self.probe_interval_seconds <= 0
or self.probe_timeout_seconds <= 0
or self.consecutive_failure_limit < 1
or self.graceful_shutdown_seconds <= 0
):
raise ValueError("Mission Core watchdog policy must be positive")
class ConsecutiveHealthGate:
"""Trigger only after a bounded sequence of genuine probe failures."""
def __init__(self, failure_limit: int) -> None:
if failure_limit < 1:
raise ValueError("health failure limit must be positive")
self.failure_limit = failure_limit
self.consecutive_failures = 0
def observe(self, healthy: bool) -> bool:
if healthy:
self.consecutive_failures = 0
return False
self.consecutive_failures += 1
return self.consecutive_failures >= self.failure_limit
class MissionCoreWatchdogJournal:
"""Append bounded, private lifecycle evidence outside the application log."""
def __init__(
self,
path: Path,
*,
max_bytes: int = DEFAULT_JOURNAL_MAX_BYTES,
) -> None:
if max_bytes < 1:
raise ValueError("watchdog journal limit must be positive")
self.path = path.expanduser().absolute()
self.max_bytes = max_bytes
self._guard = Lock()
self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
metadata = self.path.parent.lstat()
if not stat.S_ISDIR(metadata.st_mode):
raise MissionCoreWatchdogError("watchdog journal parent is not a directory")
def append(self, event: str, **details: object) -> None:
document = {
"schema_version": WATCHDOG_SCHEMA,
"event": event,
"utc_ns": time.time_ns(),
"monotonic_ns": time.monotonic_ns(),
"pid": os.getpid(),
**details,
}
payload = json.dumps(
document,
ensure_ascii=True,
separators=(",", ":"),
sort_keys=True,
).encode("utf-8") + b"\n"
with self._guard:
self._rotate_if_needed(len(payload))
flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_CLOEXEC", 0)
flags |= getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(self.path, flags, 0o600)
try:
metadata = os.fstat(descriptor)
if (
not stat.S_ISREG(metadata.st_mode)
or stat.S_IMODE(metadata.st_mode) != 0o600
or metadata.st_nlink != 1
):
raise MissionCoreWatchdogError(
"watchdog journal is not a private regular file"
)
os.write(descriptor, payload)
os.fsync(descriptor)
finally:
os.close(descriptor)
def _rotate_if_needed(self, incoming_bytes: int) -> None:
try:
metadata = self.path.lstat()
except FileNotFoundError:
return
if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1:
raise MissionCoreWatchdogError("watchdog journal identity changed")
if metadata.st_size + incoming_bytes <= self.max_bytes:
return
previous = self.path.with_name(f"{self.path.name}.1")
with suppress(FileNotFoundError):
previous.unlink()
os.replace(self.path, previous)
Probe = Callable[[], bool]
SignalAction = Callable[[], None]
class MissionCoreSelfWatchdog:
"""Terminate a live-but-unhealthy service so its init system can restart it."""
def __init__(
self,
repository_root: Path,
*,
policy: MissionCoreWatchdogPolicy | None = None,
probe: Probe | None = None,
request_shutdown: SignalAction | None = None,
force_shutdown: SignalAction | None = None,
journal: MissionCoreWatchdogJournal | None = None,
) -> None:
self.policy = policy or MissionCoreWatchdogPolicy()
self.probe = probe or _mission_core_health_probe(
self.policy.probe_timeout_seconds
)
self.request_shutdown = request_shutdown or _process_group_signal(signal.SIGTERM)
self.force_shutdown = force_shutdown or _process_group_signal(signal.SIGKILL)
self.journal = journal or MissionCoreWatchdogJournal(
repository_root.expanduser().absolute()
/ ".runtime/mission-core/service-watchdog.jsonl"
)
self._stop = Event()
self._thread = Thread(
target=self._run,
name="mission-core-self-health-watchdog",
daemon=True,
)
self._started = False
def start(self) -> None:
if self._started:
raise MissionCoreWatchdogError("Mission Core watchdog already started")
self._started = True
self.journal.append("watchdog-started", policy=_policy_dict(self.policy))
self._thread.start()
def stop(self) -> None:
if not self._started:
return
self._stop.set()
self._thread.join(timeout=self.policy.probe_timeout_seconds + 1.0)
self.journal.append(
"watchdog-stopped",
worker_alive=self._thread.is_alive(),
)
def _run(self) -> None:
if self._stop.wait(self.policy.startup_grace_seconds):
return
gate = ConsecutiveHealthGate(self.policy.consecutive_failure_limit)
last_reported_health: bool | None = None
while not self._stop.is_set():
healthy = False
try:
healthy = self.probe()
except Exception:
healthy = False
triggered = gate.observe(healthy)
if healthy != last_reported_health:
self.journal.append(
"health-state-changed",
healthy=healthy,
consecutive_failures=gate.consecutive_failures,
)
last_reported_health = healthy
if triggered:
self.journal.append(
"restart-requested",
reason="consecutive-health-probe-failures",
consecutive_failures=gate.consecutive_failures,
)
self.request_shutdown()
if not self._stop.wait(self.policy.graceful_shutdown_seconds):
self.journal.append(
"restart-escalated",
reason="graceful-shutdown-timeout",
)
self.force_shutdown()
return
if self._stop.wait(self.policy.probe_interval_seconds):
return
def watchdog_enabled(environ: Mapping[str, str] = os.environ) -> bool:
return environ.get(WATCHDOG_ENV) == WATCHDOG_ENABLED_VALUE
def _mission_core_health_probe(timeout_seconds: float) -> Probe:
def probe() -> bool:
connection = http.client.HTTPConnection(
"127.0.0.1",
8000,
timeout=timeout_seconds,
)
try:
connection.request("GET", "/api/health", headers={"Connection": "close"})
response = connection.getresponse()
payload = response.read(64 * 1024 + 1)
except (OSError, TimeoutError, http.client.HTTPException):
return False
finally:
connection.close()
if response.status != 200 or len(payload) > 64 * 1024:
return False
try:
document = json.loads(payload)
except (UnicodeDecodeError, json.JSONDecodeError):
return False
return bool(
isinstance(document, dict)
and document.get("ok") is True
and document.get("status") == "ok"
and document.get("service") == MISSION_CORE_SERVICE_ID
)
return probe
def _process_group_signal(signal_number: signal.Signals) -> SignalAction:
def send() -> None:
os.killpg(os.getpgrp(), signal_number)
return send
def _policy_dict(policy: MissionCoreWatchdogPolicy) -> dict[str, object]:
return {
"startup_grace_seconds": policy.startup_grace_seconds,
"probe_interval_seconds": policy.probe_interval_seconds,
"probe_timeout_seconds": policy.probe_timeout_seconds,
"consecutive_failure_limit": policy.consecutive_failure_limit,
"graceful_shutdown_seconds": policy.graceful_shutdown_seconds,
}
@@ -305,6 +305,7 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
decision = manifest.get("decision") decision = manifest.get("decision")
method = manifest.get("method") method = manifest.get("method")
metrics = manifest.get("metrics") metrics = manifest.get("metrics")
load_envelope = metrics.get("load_envelope") if isinstance(metrics, dict) else None
status = manifest.get("status") status = manifest.get("status")
integrated = status == INTEGRATED_STATUS integrated = status == INTEGRATED_STATUS
if ( if (
@@ -331,12 +332,35 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
or decision.get("integrated_world_state_gate_evaluated") is not integrated or decision.get("integrated_world_state_gate_evaluated") is not integrated
or (integrated and decision.get("integrated_world_state_gate_passed") is not True) or (integrated and decision.get("integrated_world_state_gate_passed") is not True)
or (integrated and decision.get("detector_replacement_authorized") is not False) or (integrated and decision.get("detector_replacement_authorized") is not False)
or (
load_envelope is not None
and (
decision.get("load_envelope_evaluated") is not True
or not isinstance(load_envelope, dict)
or decision.get("production_rate_repeatability_passed")
is not load_envelope.get("production_rate_repeatability_passed")
or decision.get("reserve_12_fps_passed")
is not load_envelope.get("reserve_12_fps_passed")
or decision.get("limit_15_fps_passed")
is not load_envelope.get("limit_15_fps_passed")
or decision.get("load_envelope_accepted")
is not load_envelope.get("load_envelope_accepted")
)
)
or decision.get("production_accepted") is not False or decision.get("production_accepted") is not False
or not isinstance(method, dict) or not isinstance(method, dict)
or method.get("schema_version") != "missioncore.laboratory-method/v1" or method.get("schema_version") != "missioncore.laboratory-method/v1"
or method.get("completeness") != "complete" or method.get("completeness") != "complete"
or not isinstance(metrics, dict) or not isinstance(metrics, dict)
or (integrated and not isinstance(metrics.get("integrated_world_state"), dict)) or (integrated and not isinstance(metrics.get("integrated_world_state"), dict))
or (
metrics.get("runtime_hardening") is not None
and not _valid_runtime_hardening(metrics["runtime_hardening"])
)
or (
metrics.get("load_envelope") is not None
and not _valid_load_envelope(metrics["load_envelope"])
)
or not isinstance(manifest.get("limitations"), list) or not isinstance(manifest.get("limitations"), list)
or not isinstance(catalog_descriptor, dict) or not isinstance(catalog_descriptor, dict)
or catalog_descriptor.get("path") != "catalog.json" or catalog_descriptor.get("path") != "catalog.json"
@@ -363,6 +387,217 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
return {"manifest": manifest, "catalog": catalog} return {"manifest": manifest, "catalog": catalog}
def _valid_runtime_hardening(value: object) -> bool:
if not isinstance(value, dict) or value.get("schema_version") != (
"missioncore.m48s-runtime-hardening-comparison/v1"
):
return False
full_run_keys = {
"source_frames_admitted",
"delivered_world_states",
"superseded_frames",
"effective_world_state_fps",
"world_state_completion_age_p95_ms",
"world_state_completion_age_p99_ms",
"world_state_completion_age_maximum_ms",
"rolling_maximum_ms",
"geometry_maximum_ms",
"additional_inference_passes",
}
for name in ("baseline", "hardened"):
run = value.get(name)
if (
not isinstance(run, dict)
or set(run) != full_run_keys
or any(not _nonnegative_number(item) for item in run.values())
):
return False
startup = value.get("startup")
if not isinstance(startup, dict) or set(startup) != {
"baseline",
"prewarmed",
"prewarm_duration_ms",
"prewarm_inference_passes",
"validation_frames",
}:
return False
for name in ("baseline", "prewarmed"):
frame = startup.get(name)
if (
not isinstance(frame, dict)
or set(frame) != {"detector_ms", "world_state_ms"}
or any(not _nonnegative_number(item) for item in frame.values())
):
return False
return (
_nonnegative_number(startup.get("prewarm_duration_ms"))
and isinstance(startup.get("prewarm_inference_passes"), int)
and not isinstance(startup.get("prewarm_inference_passes"), bool)
and startup["prewarm_inference_passes"] > 0
and isinstance(startup.get("validation_frames"), int)
and not isinstance(startup.get("validation_frames"), bool)
and startup["validation_frames"] > 0
)
def _valid_load_envelope(value: object) -> bool:
if not isinstance(value, dict):
return False
if value.get("schema_version") == "missioncore.m48s-load-envelope-comparison/v1":
return _valid_legacy_load_envelope(value)
if value.get("schema_version") != "missioncore.m48s-load-envelope-comparison/v2":
return False
if (
value.get("repeat_count_per_rate") != 3
or value.get("production_rate_repeatability_passed") is not True
or value.get("reserve_12_fps_passed") is not True
or value.get("limit_15_fps_passed") is not True
or value.get("load_envelope_accepted") is not True
or value.get("compute_capacity_at_least_fps") != 15.0
or value.get("bottleneck_interpretation")
!= "cold-video-decode-isolated-before-admission-no-steady-gpu-saturation"
):
return False
scenarios = value.get("scenarios")
scenario_ids = [item.get("id") for item in scenarios if isinstance(item, dict)] if (
isinstance(scenarios, list)
) else []
if scenario_ids != ["production-10fps", "reserve-12fps", "limit-15fps"]:
return False
required_numeric = {
"requested_source_rate_hz",
"source_frames_admitted",
"delivered_world_states",
"superseded_frames",
"delivery_ratio",
"effective_world_state_fps",
"world_state_completion_age_p95_ms",
"world_state_completion_age_p99_ms",
"world_state_completion_age_maximum_ms",
"hot_loop_decode_p95_ms",
"hot_loop_decode_maximum_ms",
"preadmission_decode_maximum_ms",
"source_prefetch_preparation_maximum_ms",
"source_pacing_lateness_p95_ms",
"source_pacing_lateness_maximum_ms",
"detector_p95_ms",
"detector_maximum_ms",
"gpu_utilization_mean_percent",
"gpu_utilization_maximum_percent",
"gpu_memory_maximum_mib",
"process_peak_rss_mib",
"queue_capacity",
"additional_inference_passes",
}
for scenario in scenarios:
repetitions = scenario.get("repetitions") if isinstance(scenario, dict) else None
if (
not isinstance(scenario, dict)
or scenario.get("repeat_count") != 3
or scenario.get("all_repetitions_passed") is not True
or scenario.get("integrity_gate_passed") is not True
or scenario.get("operating_target_gate_passed") is not True
or scenario.get("additional_inference_passes") != 0
or scenario.get("queue_capacity") != 2
or not isinstance(scenario.get("queue_high_watermarks"), dict)
or any(value != 2 for value in scenario["queue_high_watermarks"].values())
or not isinstance(scenario.get("thresholds"), dict)
or any(not _nonnegative_number(scenario.get(key)) for key in required_numeric)
or not isinstance(repetitions, list)
or len(repetitions) != 3
):
return False
for expected_repetition, repetition in enumerate(repetitions, start=1):
if (
not isinstance(repetition, dict)
or repetition.get("repetition") != expected_repetition
or repetition.get("operating_target_gate_passed") is not True
or not isinstance(repetition.get("run_id"), str)
or not repetition["run_id"]
or any(
not _nonnegative_number(repetition.get(key))
for key in {
"delivered_world_states",
"superseded_frames",
"delivery_ratio",
"effective_world_state_fps",
"world_state_completion_age_p95_ms",
"hot_loop_decode_p95_ms",
"hot_loop_decode_maximum_ms",
"preadmission_decode_maximum_ms",
"source_pacing_lateness_maximum_ms",
}
)
or any(
not isinstance(repetition.get(key), str)
or SHA256.fullmatch(repetition[key]) is None
for key in ("result_sha256", "frame_evidence_sha256")
)
):
return False
return True
def _valid_legacy_load_envelope(value: dict[str, object]) -> bool:
if (
value.get("production_rate_repeatability_passed") is not False
or value.get("reserve_12_fps_passed") is not True
or value.get("limit_15_fps_passed") is not True
or value.get("load_envelope_accepted") is not False
or value.get("compute_capacity_at_least_fps") != 15.0
or value.get("bottleneck_interpretation")
!= "rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation"
):
return False
scenarios = value.get("scenarios")
if not isinstance(scenarios, list) or [
item.get("id") for item in scenarios if isinstance(item, dict)
] != ["production-10fps", "reserve-12fps", "limit-15fps"]:
return False
expected_targets = (False, True, True)
required_numeric = {
"requested_source_rate_hz",
"source_frames_admitted",
"delivered_world_states",
"superseded_frames",
"delivery_ratio",
"effective_world_state_fps",
"world_state_completion_age_p95_ms",
"world_state_completion_age_p99_ms",
"world_state_completion_age_maximum_ms",
"decode_p95_ms",
"decode_maximum_ms",
"detector_p95_ms",
"detector_maximum_ms",
"gpu_utilization_mean_percent",
"gpu_utilization_maximum_percent",
"gpu_memory_maximum_mib",
"process_peak_rss_mib",
"queue_capacity",
"additional_inference_passes",
}
for scenario, target in zip(scenarios, expected_targets, strict=True):
if (
not isinstance(scenario, dict)
or scenario.get("integrity_gate_passed") is not True
or scenario.get("operating_target_gate_passed") is not target
or scenario.get("additional_inference_passes") != 0
or scenario.get("queue_capacity") != 2
or not isinstance(scenario.get("queue_high_watermarks"), dict)
or any(depth != 2 for depth in scenario["queue_high_watermarks"].values())
or not isinstance(scenario.get("thresholds"), dict)
or any(not _nonnegative_number(scenario.get(key)) for key in required_numeric)
or not isinstance(scenario.get("frame_evidence_sha256"), str)
or SHA256.fullmatch(scenario["frame_evidence_sha256"]) is None
):
return False
return True
def _nonnegative_number(value: object) -> bool:
return isinstance(value, (int, float)) and not isinstance(value, bool) and float(value) >= 0.0
def _candidate_signature(candidate: Path) -> tuple[int, ...]: def _candidate_signature(candidate: Path) -> tuple[int, ...]:
if not candidate.is_dir() or candidate.is_symlink(): if not candidate.is_dir() or candidate.is_symlink():
raise RuntimeError("M4.8S result candidate is invalid") raise RuntimeError("M4.8S result candidate is invalid")
+1
View File
@@ -83,6 +83,7 @@ def test_serve_resolves_frontend_from_repository_root(monkeypatch: Any) -> None:
"port": 8000, "port": 8000,
"log_level": "info", "log_level": "info",
"access_log": True, "access_log": True,
"timeout_graceful_shutdown": 10,
} }
assert lease.active is False assert lease.active is False
+46
View File
@@ -0,0 +1,46 @@
from __future__ import annotations
import plistlib
from pathlib import Path
from k1link.local_service_launchd import plan_mission_core_launch_agent
def test_launch_agent_plan_disables_sync_and_enables_watchdog(tmp_path: Path) -> None:
repository = tmp_path / "repo"
repository.mkdir()
uv_entrypoint = tmp_path / "uv"
uv_entrypoint.write_text("#!/bin/sh\n")
uv_entrypoint.chmod(0o700)
agent = tmp_path / "agent.plist"
agent.write_bytes(
plistlib.dumps(
{
"Label": "com.nodedc.mission-core.local",
"ProgramArguments": [str(uv_entrypoint), "run", "k1link", "serve"],
"WorkingDirectory": str(repository),
"EnvironmentVariables": {"PATH": "/usr/bin:/bin"},
}
)
)
agent.chmod(0o600)
plan = plan_mission_core_launch_agent(
repository_root=repository,
agent_path=agent,
)
desired = plistlib.loads(plan.desired_payload)
assert desired["ProgramArguments"] == [
str(uv_entrypoint),
"run",
"--no-sync",
"k1link",
"serve",
]
assert desired["EnvironmentVariables"]["MISSIONCORE_SERVICE_WATCHDOG"] == "1"
assert desired["KeepAlive"] is True
assert desired["RunAtLoad"] is True
assert desired["AbandonProcessGroup"] is False
assert desired["ExitTimeOut"] == 20
assert plan.current_sha256 != plan.desired_sha256
+48 -2
View File
@@ -38,7 +38,7 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N
assert manifest["result_id"] == result.result_id assert manifest["result_id"] == result.result_id
assert result.result_id.endswith(identity_digest) assert result.result_id.endswith(identity_digest)
assert manifest["identity_sha256"] == identity_digest assert manifest["identity_sha256"] == identity_digest
assert len(manifest["artifacts"]) == 31 assert len(manifest["artifacts"]) == 45
assert manifest["method"]["completeness"] == "complete" assert manifest["method"]["completeness"] == "complete"
assert manifest["bounded_question_accepted"] is True assert manifest["bounded_question_accepted"] is True
assert manifest["ground_truth"] is False assert manifest["ground_truth"] is False
@@ -49,6 +49,11 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N
"integrated_world_state_gate_evaluated": True, "integrated_world_state_gate_evaluated": True,
"integrated_world_state_gate_passed": True, "integrated_world_state_gate_passed": True,
"full_replay_visual_published": True, "full_replay_visual_published": True,
"load_envelope_evaluated": True,
"production_rate_repeatability_passed": True,
"reserve_12_fps_passed": True,
"limit_15_fps_passed": True,
"load_envelope_accepted": True,
"detector_replacement_authorized": False, "detector_replacement_authorized": False,
"production_accepted": False, "production_accepted": False,
} }
@@ -66,6 +71,47 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N
assert max(integrated["queue_high_watermarks"].values()) <= 2 assert max(integrated["queue_high_watermarks"].values()) <= 2
assert integrated["additional_inference_passes"] == 0 assert integrated["additional_inference_passes"] == 0
assert integrated["failures"] == 0 assert integrated["failures"] == 0
hardening = manifest["metrics"]["runtime_hardening"]
assert hardening["schema_version"] == "missioncore.m48s-runtime-hardening-comparison/v1"
assert hardening["baseline"]["delivered_world_states"] == 4_480
assert hardening["baseline"]["superseded_frames"] == 9
assert hardening["hardened"]["delivered_world_states"] == 4_488
assert hardening["hardened"]["superseded_frames"] == 1
assert hardening["baseline"]["rolling_maximum_ms"] == 762.638263
assert hardening["hardened"]["rolling_maximum_ms"] == 28.044958
assert hardening["startup"]["baseline"]["detector_ms"] == 397.356888
assert hardening["startup"]["prewarmed"]["detector_ms"] == 27.662887
assert hardening["startup"]["baseline"]["world_state_ms"] == 449.893287
assert hardening["startup"]["prewarmed"]["world_state_ms"] == 60.516957
assert hardening["startup"]["prewarm_duration_ms"] == 420.721918
assert hardening["startup"]["validation_frames"] == 1_000
envelope = manifest["metrics"]["load_envelope"]
assert envelope["schema_version"] == "missioncore.m48s-load-envelope-comparison/v2"
assert envelope["repeat_count_per_rate"] == 3
assert envelope["production_rate_repeatability_passed"] is True
assert envelope["reserve_12_fps_passed"] is True
assert envelope["limit_15_fps_passed"] is True
assert envelope["load_envelope_accepted"] is True
assert envelope["compute_capacity_at_least_fps"] == 15.0
production, reserve, limit = envelope["scenarios"]
assert production["delivered_world_states"] >= 4_485
assert production["superseded_frames"] <= 5
assert production["operating_target_gate_passed"] is True
assert reserve["delivered_world_states"] >= 4_467
assert reserve["superseded_frames"] <= 22
assert reserve["operating_target_gate_passed"] is True
assert limit["delivered_world_states"] == 4_488
assert limit["superseded_frames"] == 1
assert limit["operating_target_gate_passed"] is True
assert all(item["repeat_count"] == 3 for item in envelope["scenarios"])
assert all(len(item["repetitions"]) == 3 for item in envelope["scenarios"])
assert all(item["all_repetitions_passed"] is True for item in envelope["scenarios"])
assert all(item["hot_loop_decode_p95_ms"] < 30 for item in envelope["scenarios"])
assert all(
item["source_prefetch_preparation_maximum_ms"] > 3_000
for item in envelope["scenarios"]
)
assert all(item["additional_inference_passes"] == 0 for item in envelope["scenarios"])
catalog = json.loads((result.result_root / "catalog.json").read_text("utf-8")) catalog = json.loads((result.result_root / "catalog.json").read_text("utf-8"))
assert catalog["frame_count"] == len(FRAME_IDS) == 11 assert catalog["frame_count"] == len(FRAME_IDS) == 11
@@ -85,7 +131,7 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N
) )
proof = verify_laboratory_evidence_result(definition, result.result_root) proof = verify_laboratory_evidence_result(definition, result.result_root)
assert proof["result_id"] == result.result_id assert proof["result_id"] == result.result_id
assert proof["artifact_count"] == 31 assert proof["artifact_count"] == 45
with pytest.raises(M48SFixedClassDetectorLabError, match="already exists"): with pytest.raises(M48SFixedClassDetectorLabError, match="already exists"):
build_m48s_fixed_class_detector_lab( build_m48s_fixed_class_detector_lab(
+19 -13
View File
@@ -47,20 +47,30 @@ def test_m48s_lab_api_projects_verified_result_frame_and_camera(tmp_path: Path)
result.json()["metrics"]["integrated_world_state"]["world_state_completion_age_p95_ms"] result.json()["metrics"]["integrated_world_state"]["world_state_completion_age_p95_ms"]
== 74.733648 == 74.733648
) )
hardening = result.json()["metrics"]["runtime_hardening"]
assert hardening["baseline"]["delivered_world_states"] == 4_480
assert hardening["hardened"]["delivered_world_states"] == 4_488
assert hardening["startup"]["prewarmed"]["world_state_ms"] == 60.516957
envelope = result.json()["metrics"]["load_envelope"]
assert envelope["production_rate_repeatability_passed"] is True
assert envelope["reserve_12_fps_passed"] is True
assert envelope["limit_15_fps_passed"] is True
assert envelope["load_envelope_accepted"] is True
assert envelope["repeat_count_per_rate"] == 3
assert [item["requested_source_rate_hz"] for item in envelope["scenarios"]] == [
10.0,
12.0,
15.0,
]
assert result.json()["ground_truth"] is False assert result.json()["ground_truth"] is False
assert len(result.json()["frames"]) == 11 assert len(result.json()["frames"]) == 11
timeline = client.get( timeline = client.get(f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/timeline")
f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/timeline"
)
assert timeline.status_code == 200 assert timeline.status_code == 200
assert timeline.json()["frame_count"] == 4_489 assert timeline.json()["frame_count"] == 4_489
assert timeline.json()["world_state_frame_count"] == 4_481 assert timeline.json()["world_state_frame_count"] == 4_481
assert timeline.json()["superseded_frame_count"] == 8 assert timeline.json()["superseded_frame_count"] == 8
assert ( assert timeline.json()["camera_point_delivery"] == "factory-kb4-causal-registered-accumulation"
timeline.json()["camera_point_delivery"]
== "factory-kb4-causal-registered-accumulation"
)
assert timeline.json()["camera_point_window_seconds"] == 2.0 assert timeline.json()["camera_point_window_seconds"] == 2.0
assert timeline.json()["camera_point_sample_limit"] == 20_000 assert timeline.json()["camera_point_sample_limit"] == 20_000
chunk = client.get( chunk = client.get(
@@ -72,9 +82,7 @@ def test_m48s_lab_api_projects_verified_result_frame_and_camera(tmp_path: Path)
assert replay_frame["world_state_available"] is True assert replay_frame["world_state_available"] is True
assert replay_frame["camera_projection"] == "factory-kb4-exact" assert replay_frame["camera_projection"] == "factory-kb4-exact"
assert replay_frame["camera_projected_sample_count"] > 0 assert replay_frame["camera_projected_sample_count"] > 0
assert any( assert any(item["semantic_hint"] == "dog" for item in replay_frame["camera_proposals"])
item["semantic_hint"] == "dog" for item in replay_frame["camera_proposals"]
)
camera_points = client.get( camera_points = client.get(
f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}" f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}"
"/timeline/frames/253/camera-points" "/timeline/frames/253/camera-points"
@@ -82,9 +90,7 @@ def test_m48s_lab_api_projects_verified_result_frame_and_camera(tmp_path: Path)
assert camera_points.status_code == 200 assert camera_points.status_code == 200
camera_point_payload = camera_points.json() camera_point_payload = camera_points.json()
assert camera_point_payload["schema_version"] == "missioncore.m48s-camera-point-overlay/v1" assert camera_point_payload["schema_version"] == "missioncore.m48s-camera-point-overlay/v1"
assert camera_point_payload["projection"] == ( assert camera_point_payload["projection"] == ("factory-kb4-causal-registered-accumulation")
"factory-kb4-causal-registered-accumulation"
)
assert camera_point_payload["source_frame_count"] > 1 assert camera_point_payload["source_frame_count"] > 1
assert camera_point_payload["sample_count"] > replay_frame["camera_projected_sample_count"] assert camera_point_payload["sample_count"] > replay_frame["camera_projected_sample_count"]
assert camera_point_payload["sample_count"] <= 20_000 assert camera_point_payload["sample_count"] <= 20_000
+18 -1
View File
@@ -6,7 +6,7 @@ from pathlib import Path
from unittest.mock import patch from unittest.mock import patch
from k1link.perception.detector import DetectorFrameTiming from k1link.perception.detector import DetectorFrameTiming
from k1link.perception.recorded_source import DecodedFrameTiming from k1link.perception.recorded_source import DecodedFrameTiming, SourcePacingTiming
REPOSITORY_ROOT = Path(__file__).resolve().parents[1] REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
RUNNER_PATH = REPOSITORY_ROOT / "experiments/perception/run_m48s_reference_graph_shadow_worker.py" RUNNER_PATH = REPOSITORY_ROOT / "experiments/perception/run_m48s_reference_graph_shadow_worker.py"
@@ -19,6 +19,14 @@ SPEC.loader.exec_module(RUNNER)
def test_frame_timing_store_closes_provider_and_unattributed_time() -> None: def test_frame_timing_store_closes_provider_and_unattributed_time() -> None:
store = RUNNER.FrameTimingStore() store = RUNNER.FrameTimingStore()
store.observe_decode(DecodedFrameTiming(sequence=7, duration_ns=5)) store.observe_decode(DecodedFrameTiming(sequence=7, duration_ns=5))
store.observe_pacing(
SourcePacingTiming(
sequence=7,
scheduled_monotonic_ns=80,
emitted_monotonic_ns=90,
lateness_ns=10,
)
)
store.observe_detector( store.observe_detector(
DetectorFrameTiming( DetectorFrameTiming(
sequence=7, sequence=7,
@@ -50,6 +58,14 @@ def test_frame_timing_store_closes_provider_and_unattributed_time() -> None:
def test_pipeline_timing_metrics_preserve_single_pass_stage_breakdown() -> None: def test_pipeline_timing_metrics_preserve_single_pass_stage_breakdown() -> None:
store = RUNNER.FrameTimingStore() store = RUNNER.FrameTimingStore()
store.observe_decode(DecodedFrameTiming(sequence=3, duration_ns=1_000_000)) store.observe_decode(DecodedFrameTiming(sequence=3, duration_ns=1_000_000))
store.observe_pacing(
SourcePacingTiming(
sequence=3,
scheduled_monotonic_ns=0,
emitted_monotonic_ns=1_000_000,
lateness_ns=1_000_000,
)
)
store.observe_detector( store.observe_detector(
DetectorFrameTiming( DetectorFrameTiming(
sequence=3, sequence=3,
@@ -79,6 +95,7 @@ def test_pipeline_timing_metrics_preserve_single_pass_stage_breakdown() -> None:
assert metrics["detector_ms"]["inference_transport"]["maximum"] == 3.0 assert metrics["detector_ms"]["inference_transport"]["maximum"] == 3.0
assert metrics["provider_ms"]["threat"]["maximum"] == 5.0 assert metrics["provider_ms"]["threat"]["maximum"] == 5.0
assert metrics["graph_unattributed_ms"]["maximum"] == 6.0 assert metrics["graph_unattributed_ms"]["maximum"] == 6.0
assert metrics["delivered_source_pacing_lateness_ms"]["maximum"] == 1.0
assert metrics["maximum_graph_sequence"] == 3 assert metrics["maximum_graph_sequence"] == 3
assert metrics["additional_inference_passes"] == 0 assert metrics["additional_inference_passes"] == 0
@@ -0,0 +1,63 @@
from __future__ import annotations
import importlib.util
import subprocess
from pathlib import Path
from typing import Any
_SCRIPT = Path(__file__).parents[1] / "scripts/manage_mission_core_launch_agent.py"
_SPEC = importlib.util.spec_from_file_location(
"manage_mission_core_launch_agent",
_SCRIPT,
)
assert _SPEC is not None and _SPEC.loader is not None
manager = importlib.util.module_from_spec(_SPEC)
_SPEC.loader.exec_module(manager)
def test_reload_waits_for_launchd_transition_before_bootstrap(
monkeypatch: Any,
tmp_path: Path,
) -> None:
commands: list[tuple[str, ...]] = []
print_results = iter((0, 0, 1))
def fake_run(arguments: list[str], **_: object) -> subprocess.CompletedProcess[str]:
command = tuple(arguments)
commands.append(command)
if arguments[1] == "print":
return subprocess.CompletedProcess(arguments, next(print_results), "", "")
return subprocess.CompletedProcess(arguments, 0, "", "")
monkeypatch.setattr(manager.subprocess, "run", fake_run)
monkeypatch.setattr(manager.time, "sleep", lambda _: None)
agent = tmp_path / "agent.plist"
manager._reload_launch_agent(agent)
assert [command[1] for command in commands] == [
"bootout",
"print",
"print",
"print",
"bootstrap",
]
def test_reload_rejects_failed_bootout_while_job_is_still_loaded(
monkeypatch: Any,
tmp_path: Path,
) -> None:
def fake_run(arguments: list[str], **_: object) -> subprocess.CompletedProcess[str]:
if arguments[1] == "bootout":
return subprocess.CompletedProcess(arguments, 5, "", "")
return subprocess.CompletedProcess(arguments, 0, "", "")
monkeypatch.setattr(manager.subprocess, "run", fake_run)
try:
manager._reload_launch_agent(tmp_path / "agent.plist")
except manager.MissionCoreLaunchAgentError as exc:
assert "bootout failed with exit code 5" in str(exc)
else:
raise AssertionError("failed bootout was accepted")
+89
View File
@@ -62,9 +62,12 @@ from k1link.perception.providers import (
from k1link.perception.recorded_source import ( from k1link.perception.recorded_source import (
DecodedFrameTiming, DecodedFrameTiming,
DecodedRecordedSource, DecodedRecordedSource,
DecodePhase,
PrefetchedRecordedImageDecoder,
RecordedRavnoves00Source, RecordedRavnoves00Source,
RecordedSourceError, RecordedSourceError,
ReplayPacing, ReplayPacing,
SourcePacingTiming,
) )
@@ -835,6 +838,92 @@ def test_recorded_source_reuses_one_timeline_for_1x_and_uncapped(tmp_path: Path)
assert one_x[1].registered_point_increment_payload is None assert one_x[1].registered_point_increment_payload is None
def test_recorded_source_target_rate_changes_only_wall_clock_pacing(tmp_path: Path) -> None:
camera_path, timeline_path = _write_recorded_fixture(tmp_path)
now = [1_000_000_000]
waits: list[float] = []
pacing: list[SourcePacingTiming] = []
def wait(stop_event: Event, seconds: float) -> bool:
waits.append(seconds)
now[0] += round(seconds * 1_000_000_000)
return stop_event.is_set()
source = RecordedRavnoves00Source(
camera_index_path=camera_path,
source_pack_path=timeline_path,
pacing=ReplayPacing.ONE_X,
target_rate_hz=20.0,
expected_frame_count=2,
expected_source_pack_sha256=None,
pacing_observer=pacing.append,
clock_ns=lambda: now[0],
wait=wait,
)
packets = list(source.packets(Event()))
assert waits == pytest.approx([0.05])
assert [sample.sequence for sample in pacing] == [0, 1]
assert [sample.lateness_ns for sample in pacing] == [0, 0]
assert (
packets[1].envelope.timestamps.source_ns
- packets[0].envelope.timestamps.source_ns
== 100_000_000
)
def test_recorded_source_rejects_target_rate_for_uncapped_replay(tmp_path: Path) -> None:
camera_path, timeline_path = _write_recorded_fixture(tmp_path)
with pytest.raises(RecordedSourceError, match="requires paced replay"):
RecordedRavnoves00Source(
camera_index_path=camera_path,
source_pack_path=timeline_path,
pacing=ReplayPacing.UNCAPPED,
target_rate_hz=12.0,
expected_frame_count=2,
expected_source_pack_sha256=None,
)
def test_prefetched_decoder_moves_cold_decode_before_source_admission() -> None:
observed: list[DecodedFrameTiming] = []
class Decoder:
def frames(self, stop_event: Event) -> Iterator[np.ndarray]:
for index in range(4):
if stop_event.is_set():
return
yield np.full((2, 3, 3), index, dtype=np.uint8)
decoder = PrefetchedRecordedImageDecoder(
Decoder(),
capacity_frames=2,
ready_frames=2,
timing_observer=observed.append,
)
stop_event = Event()
snapshot = decoder.prepare(stop_event)
assert snapshot.buffered_frames == 2
assert snapshot.capacity_frames == 2
assert [sample.phase for sample in observed] == [
DecodePhase.PREADMISSION,
DecodePhase.PREADMISSION,
]
decoder.mark_admission_started()
frames = list(decoder.frames(stop_event))
decoder.close()
assert [int(frame[0, 0, 0]) for frame in frames] == [0, 1, 2, 3]
assert [sample.sequence for sample in observed] == [0, 1, 2, 3]
assert [sample.phase for sample in observed[2:]] == [
DecodePhase.HOT_LOOP,
DecodePhase.HOT_LOOP,
]
def test_recorded_source_rejects_timeline_mismatch(tmp_path: Path) -> None: def test_recorded_source_rejects_timeline_mismatch(tmp_path: Path) -> None:
camera_path, timeline_path = _write_recorded_fixture(tmp_path, mismatched=True) camera_path, timeline_path = _write_recorded_fixture(tmp_path, mismatched=True)
source = RecordedRavnoves00Source( source = RecordedRavnoves00Source(
+116
View File
@@ -0,0 +1,116 @@
from __future__ import annotations
import json
from pathlib import Path
from threading import Event
from k1link.service_watchdog import (
ConsecutiveHealthGate,
MissionCoreSelfWatchdog,
MissionCoreWatchdogJournal,
MissionCoreWatchdogPolicy,
watchdog_enabled,
)
def test_consecutive_health_gate_resets_after_recovery() -> None:
gate = ConsecutiveHealthGate(3)
assert gate.observe(False) is False
assert gate.observe(False) is False
assert gate.observe(True) is False
assert gate.consecutive_failures == 0
assert gate.observe(False) is False
assert gate.observe(False) is False
assert gate.observe(False) is True
def test_self_watchdog_escalates_a_persistently_unhealthy_process(tmp_path: Path) -> None:
requested = Event()
forced = Event()
journal_path = tmp_path / "watchdog.jsonl"
watchdog = MissionCoreSelfWatchdog(
tmp_path,
policy=MissionCoreWatchdogPolicy(
startup_grace_seconds=0.01,
probe_interval_seconds=0.01,
probe_timeout_seconds=0.01,
consecutive_failure_limit=2,
graceful_shutdown_seconds=0.02,
),
probe=lambda: False,
request_shutdown=requested.set,
force_shutdown=forced.set,
journal=MissionCoreWatchdogJournal(journal_path),
)
watchdog.start()
assert requested.wait(0.5)
assert forced.wait(0.5)
watchdog.stop()
events = [json.loads(line)["event"] for line in journal_path.read_text().splitlines()]
assert events == [
"watchdog-started",
"health-state-changed",
"restart-requested",
"restart-escalated",
"watchdog-stopped",
]
def test_self_watchdog_leaves_a_healthy_process_running(tmp_path: Path) -> None:
probed = Event()
requested = Event()
forced = Event()
journal_path = tmp_path / "watchdog.jsonl"
def healthy_probe() -> bool:
probed.set()
return True
watchdog = MissionCoreSelfWatchdog(
tmp_path,
policy=MissionCoreWatchdogPolicy(
startup_grace_seconds=0.01,
probe_interval_seconds=0.01,
probe_timeout_seconds=0.01,
consecutive_failure_limit=2,
graceful_shutdown_seconds=0.02,
),
probe=healthy_probe,
request_shutdown=requested.set,
force_shutdown=forced.set,
journal=MissionCoreWatchdogJournal(journal_path),
)
watchdog.start()
assert probed.wait(0.5)
watchdog.stop()
assert requested.is_set() is False
assert forced.is_set() is False
events = [json.loads(line)["event"] for line in journal_path.read_text().splitlines()]
assert events == [
"watchdog-started",
"health-state-changed",
"watchdog-stopped",
]
def test_watchdog_journal_rotates_before_exceeding_bound(tmp_path: Path) -> None:
path = tmp_path / "watchdog.jsonl"
journal = MissionCoreWatchdogJournal(path, max_bytes=300)
journal.append("first", payload="x" * 180)
journal.append("second", payload="y" * 180)
assert path.is_file()
assert path.with_name("watchdog.jsonl.1").is_file()
assert json.loads(path.read_text())["event"] == "second"
def test_watchdog_requires_exact_enable_marker() -> None:
assert watchdog_enabled({"MISSIONCORE_SERVICE_WATCHDOG": "1"}) is True
assert watchdog_enabled({"MISSIONCORE_SERVICE_WATCHDOG": "true"}) is False
assert watchdog_enabled({}) is False