Compare commits
6
Commits
84624ae3ea
...
209d0bfc26
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
209d0bfc26 | ||
|
|
f50e0077c5 | ||
|
|
fb5bf943c9 | ||
|
|
bab0bb2df2 | ||
|
|
e21332b9e1 | ||
|
|
078b8421a6 |
@@ -2,6 +2,7 @@ import type { LaboratoryFetch } from "./advancedResults";
|
||||
|
||||
const RESULT_ID = /^m48s-fixed-class-detector-lab-[a-f0-9]{64}$/;
|
||||
const FRAME_ID = /^[0-9]{6}$/;
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
const MODES = ["source", "yolox", "dfine", "rf-detr"] as const;
|
||||
const DETECTOR_MODES = ["yolox", "dfine", "rf-detr"] as const;
|
||||
const RESULT_STATUSES = [
|
||||
@@ -80,6 +81,100 @@ export interface M48SIntegratedWorldState {
|
||||
failures: number;
|
||||
}
|
||||
|
||||
export interface M48SRuntimeHardeningFullRun {
|
||||
sourceFramesAdmitted: number;
|
||||
deliveredWorldStates: number;
|
||||
supersededFrames: number;
|
||||
effectiveWorldStateFps: number;
|
||||
worldStateCompletionAgeP95Ms: number;
|
||||
worldStateCompletionAgeP99Ms: number;
|
||||
worldStateCompletionAgeMaximumMs: number;
|
||||
rollingMaximumMs: number;
|
||||
geometryMaximumMs: number;
|
||||
additionalInferencePasses: number;
|
||||
}
|
||||
|
||||
export interface M48SRuntimeHardeningComparison {
|
||||
baseline: M48SRuntimeHardeningFullRun;
|
||||
hardened: M48SRuntimeHardeningFullRun;
|
||||
startup: {
|
||||
baseline: { detectorMs: number; worldStateMs: number };
|
||||
prewarmed: { detectorMs: number; worldStateMs: number };
|
||||
prewarmDurationMs: number;
|
||||
prewarmInferencePasses: number;
|
||||
validationFrames: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface M48SLoadEnvelopeScenario {
|
||||
id: "production-10fps" | "reserve-12fps" | "limit-15fps";
|
||||
loadPurpose: "production-rate" | "reserve-gate" | "limit-discovery";
|
||||
requestedSourceRateHz: number;
|
||||
sourceFramesAdmitted: number;
|
||||
deliveredWorldStates: number;
|
||||
supersededFrames: number;
|
||||
deliveryRatio: number;
|
||||
effectiveWorldStateFps: number;
|
||||
worldStateCompletionAgeP95Ms: number;
|
||||
worldStateCompletionAgeP99Ms: number;
|
||||
worldStateCompletionAgeMaximumMs: number;
|
||||
hotLoopDecodeP95Ms: number;
|
||||
hotLoopDecodeMaximumMs: number;
|
||||
preadmissionDecodeMaximumMs: number;
|
||||
sourcePrefetchPreparationMaximumMs: number;
|
||||
sourcePacingLatenessP95Ms: number;
|
||||
sourcePacingLatenessMaximumMs: number;
|
||||
detectorP95Ms: number;
|
||||
detectorMaximumMs: number;
|
||||
gpuUtilizationMeanPercent: number;
|
||||
gpuUtilizationMaximumPercent: number;
|
||||
gpuMemoryMaximumMib: number;
|
||||
processPeakRssMib: number;
|
||||
queueHighWatermarks: Readonly<Record<"detector" | "geometry" | "temporal" | "rolling" | "threat", number>>;
|
||||
queueCapacity: number;
|
||||
additionalInferencePasses: number;
|
||||
repeatCount: number;
|
||||
allRepetitionsPassed: boolean;
|
||||
integrityGatePassed: boolean;
|
||||
operatingTargetGatePassed: boolean;
|
||||
thresholds: {
|
||||
minimumDeliveryRatio: number;
|
||||
minimumEffectiveWorldStateFps: number;
|
||||
maximumWorldStateCompletionP95Ms: number;
|
||||
};
|
||||
repetitions: readonly M48SLoadEnvelopeRepetition[];
|
||||
}
|
||||
|
||||
export interface M48SLoadEnvelopeRepetition {
|
||||
runId: string;
|
||||
repetition: 1 | 2 | 3;
|
||||
deliveredWorldStates: number;
|
||||
supersededFrames: number;
|
||||
deliveryRatio: number;
|
||||
effectiveWorldStateFps: number;
|
||||
worldStateCompletionAgeP95Ms: number;
|
||||
hotLoopDecodeP95Ms: number;
|
||||
hotLoopDecodeMaximumMs: number;
|
||||
preadmissionDecodeMaximumMs: number;
|
||||
sourcePacingLatenessMaximumMs: number;
|
||||
operatingTargetGatePassed: boolean;
|
||||
resultSha256: string;
|
||||
frameEvidenceSha256: string;
|
||||
}
|
||||
|
||||
export interface M48SLoadEnvelopeComparison {
|
||||
repeatCountPerRate: number;
|
||||
productionRateRepeatabilityPassed: boolean;
|
||||
reserve12FpsPassed: boolean;
|
||||
limit15FpsPassed: boolean;
|
||||
loadEnvelopeAccepted: boolean;
|
||||
computeCapacityAtLeastFps: 15;
|
||||
bottleneckInterpretation:
|
||||
| "rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation"
|
||||
| "cold-video-decode-isolated-before-admission-no-steady-gpu-saturation";
|
||||
scenarios: readonly M48SLoadEnvelopeScenario[];
|
||||
}
|
||||
|
||||
export interface M48SFixedClassDetectorResult {
|
||||
resultId: string;
|
||||
createdAtUtc: string;
|
||||
@@ -118,6 +213,8 @@ export interface M48SFixedClassDetectorResult {
|
||||
failures: number;
|
||||
};
|
||||
integratedWorldState: M48SIntegratedWorldState | null;
|
||||
runtimeHardening: M48SRuntimeHardeningComparison | null;
|
||||
loadEnvelope: M48SLoadEnvelopeComparison | null;
|
||||
};
|
||||
decision: {
|
||||
selectedCandidate: "rf-detr";
|
||||
@@ -125,6 +222,11 @@ export interface M48SFixedClassDetectorResult {
|
||||
integratedWorldStateGateEvaluated: boolean;
|
||||
integratedWorldStateGatePassed: boolean;
|
||||
detectorReplacementAuthorized: false;
|
||||
loadEnvelopeEvaluated: boolean;
|
||||
productionRateRepeatabilityPassed: boolean;
|
||||
reserve12FpsPassed: boolean;
|
||||
limit15FpsPassed: boolean;
|
||||
loadEnvelopeAccepted: boolean;
|
||||
productionAccepted: false;
|
||||
};
|
||||
limitations: readonly string[];
|
||||
@@ -366,6 +468,288 @@ function integratedWorldStateValue(value: unknown): M48SIntegratedWorldState {
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeHardeningFullRunValue(
|
||||
value: unknown,
|
||||
label: string,
|
||||
): M48SRuntimeHardeningFullRun {
|
||||
const run = objectValue(value, label);
|
||||
return {
|
||||
sourceFramesAdmitted: integerValue(run.source_frames_admitted, `${label}.source_frames_admitted`),
|
||||
deliveredWorldStates: integerValue(run.delivered_world_states, `${label}.delivered_world_states`),
|
||||
supersededFrames: integerValue(run.superseded_frames, `${label}.superseded_frames`),
|
||||
effectiveWorldStateFps: numberValue(run.effective_world_state_fps, `${label}.effective_world_state_fps`),
|
||||
worldStateCompletionAgeP95Ms: numberValue(run.world_state_completion_age_p95_ms, `${label}.world_state_completion_age_p95_ms`),
|
||||
worldStateCompletionAgeP99Ms: numberValue(run.world_state_completion_age_p99_ms, `${label}.world_state_completion_age_p99_ms`),
|
||||
worldStateCompletionAgeMaximumMs: numberValue(run.world_state_completion_age_maximum_ms, `${label}.world_state_completion_age_maximum_ms`),
|
||||
rollingMaximumMs: numberValue(run.rolling_maximum_ms, `${label}.rolling_maximum_ms`),
|
||||
geometryMaximumMs: numberValue(run.geometry_maximum_ms, `${label}.geometry_maximum_ms`),
|
||||
additionalInferencePasses: integerValue(run.additional_inference_passes, `${label}.additional_inference_passes`),
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeHardeningComparisonValue(value: unknown): M48SRuntimeHardeningComparison {
|
||||
const comparison = objectValue(value, "M4.8S.metrics.runtime_hardening");
|
||||
exact(
|
||||
comparison.schema_version,
|
||||
"missioncore.m48s-runtime-hardening-comparison/v1",
|
||||
"M4.8S.metrics.runtime_hardening.schema_version",
|
||||
);
|
||||
const startup = objectValue(comparison.startup, "M4.8S.runtime_hardening.startup");
|
||||
const startupFrame = (
|
||||
candidate: unknown,
|
||||
label: string,
|
||||
): { detectorMs: number; worldStateMs: number } => {
|
||||
const frame = objectValue(candidate, label);
|
||||
return {
|
||||
detectorMs: numberValue(frame.detector_ms, `${label}.detector_ms`),
|
||||
worldStateMs: numberValue(frame.world_state_ms, `${label}.world_state_ms`),
|
||||
};
|
||||
};
|
||||
return {
|
||||
baseline: runtimeHardeningFullRunValue(
|
||||
comparison.baseline,
|
||||
"M4.8S.runtime_hardening.baseline",
|
||||
),
|
||||
hardened: runtimeHardeningFullRunValue(
|
||||
comparison.hardened,
|
||||
"M4.8S.runtime_hardening.hardened",
|
||||
),
|
||||
startup: {
|
||||
baseline: startupFrame(startup.baseline, "M4.8S.runtime_hardening.startup.baseline"),
|
||||
prewarmed: startupFrame(startup.prewarmed, "M4.8S.runtime_hardening.startup.prewarmed"),
|
||||
prewarmDurationMs: numberValue(
|
||||
startup.prewarm_duration_ms,
|
||||
"M4.8S.runtime_hardening.startup.prewarm_duration_ms",
|
||||
),
|
||||
prewarmInferencePasses: integerValue(
|
||||
startup.prewarm_inference_passes,
|
||||
"M4.8S.runtime_hardening.startup.prewarm_inference_passes",
|
||||
),
|
||||
validationFrames: integerValue(
|
||||
startup.validation_frames,
|
||||
"M4.8S.runtime_hardening.startup.validation_frames",
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function loadEnvelopeComparisonValue(value: unknown): M48SLoadEnvelopeComparison {
|
||||
const comparison = objectValue(value, "M4.8S.metrics.load_envelope");
|
||||
if (comparison.schema_version === "missioncore.m48s-load-envelope-comparison/v1") {
|
||||
exact(comparison.production_rate_repeatability_passed, false, "M4.8S.load_envelope.production_rate_repeatability_passed");
|
||||
exact(comparison.reserve_12_fps_passed, true, "M4.8S.load_envelope.reserve_12_fps_passed");
|
||||
exact(comparison.limit_15_fps_passed, true, "M4.8S.load_envelope.limit_15_fps_passed");
|
||||
exact(comparison.load_envelope_accepted, false, "M4.8S.load_envelope.load_envelope_accepted");
|
||||
exact(comparison.compute_capacity_at_least_fps, 15, "M4.8S.load_envelope.compute_capacity_at_least_fps");
|
||||
exact(
|
||||
comparison.bottleneck_interpretation,
|
||||
"rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation",
|
||||
"M4.8S.load_envelope.bottleneck_interpretation",
|
||||
);
|
||||
const ids = ["production-10fps", "reserve-12fps", "limit-15fps"] as const;
|
||||
const purposes = ["production-rate", "reserve-gate", "limit-discovery"] as const;
|
||||
const targetGates = [false, true, true] as const;
|
||||
const scenarios = arrayValue(comparison.scenarios, "M4.8S.load_envelope.scenarios").map((value, index) => {
|
||||
const label = `M4.8S.load_envelope.scenarios[${index}]`;
|
||||
const scenario = objectValue(value, label);
|
||||
const id = ids[index];
|
||||
const purpose = purposes[index];
|
||||
const targetGate = targetGates[index];
|
||||
if (id === undefined || purpose === undefined || targetGate === undefined) {
|
||||
throw new M48SFixedClassDetectorContractError("M4.8S.load_envelope.scenarios: лишний legacy-сценарий.");
|
||||
}
|
||||
exact(scenario.id, id, `${label}.id`);
|
||||
exact(scenario.load_purpose, purpose, `${label}.load_purpose`);
|
||||
exact(scenario.integrity_gate_passed, true, `${label}.integrity_gate_passed`);
|
||||
exact(scenario.operating_target_gate_passed, targetGate, `${label}.operating_target_gate_passed`);
|
||||
exact(scenario.additional_inference_passes, 0, `${label}.additional_inference_passes`);
|
||||
const queues = integerRecordValue(scenario.queue_high_watermarks, `${label}.queue_high_watermarks`);
|
||||
const thresholds = objectValue(scenario.thresholds, `${label}.thresholds`);
|
||||
const frameEvidenceSha256 = textValue(scenario.frame_evidence_sha256, `${label}.frame_evidence_sha256`);
|
||||
if (!SHA256.test(frameEvidenceSha256)) {
|
||||
throw new M48SFixedClassDetectorContractError(`${label}.frame_evidence_sha256: нарушен SHA-256.`);
|
||||
}
|
||||
const legacyDecodeP95 = numberValue(scenario.decode_p95_ms, `${label}.decode_p95_ms`);
|
||||
const legacyDecodeMaximum = numberValue(scenario.decode_maximum_ms, `${label}.decode_maximum_ms`);
|
||||
return {
|
||||
id,
|
||||
loadPurpose: purpose,
|
||||
requestedSourceRateHz: numberValue(scenario.requested_source_rate_hz, `${label}.requested_source_rate_hz`),
|
||||
sourceFramesAdmitted: integerValue(scenario.source_frames_admitted, `${label}.source_frames_admitted`),
|
||||
deliveredWorldStates: integerValue(scenario.delivered_world_states, `${label}.delivered_world_states`),
|
||||
supersededFrames: integerValue(scenario.superseded_frames, `${label}.superseded_frames`),
|
||||
deliveryRatio: numberValue(scenario.delivery_ratio, `${label}.delivery_ratio`),
|
||||
effectiveWorldStateFps: numberValue(scenario.effective_world_state_fps, `${label}.effective_world_state_fps`),
|
||||
worldStateCompletionAgeP95Ms: numberValue(scenario.world_state_completion_age_p95_ms, `${label}.world_state_completion_age_p95_ms`),
|
||||
worldStateCompletionAgeP99Ms: numberValue(scenario.world_state_completion_age_p99_ms, `${label}.world_state_completion_age_p99_ms`),
|
||||
worldStateCompletionAgeMaximumMs: numberValue(scenario.world_state_completion_age_maximum_ms, `${label}.world_state_completion_age_maximum_ms`),
|
||||
hotLoopDecodeP95Ms: legacyDecodeP95,
|
||||
hotLoopDecodeMaximumMs: legacyDecodeMaximum,
|
||||
preadmissionDecodeMaximumMs: legacyDecodeMaximum,
|
||||
sourcePrefetchPreparationMaximumMs: 0,
|
||||
sourcePacingLatenessP95Ms: 0,
|
||||
sourcePacingLatenessMaximumMs: 0,
|
||||
detectorP95Ms: numberValue(scenario.detector_p95_ms, `${label}.detector_p95_ms`),
|
||||
detectorMaximumMs: numberValue(scenario.detector_maximum_ms, `${label}.detector_maximum_ms`),
|
||||
gpuUtilizationMeanPercent: numberValue(scenario.gpu_utilization_mean_percent, `${label}.gpu_utilization_mean_percent`),
|
||||
gpuUtilizationMaximumPercent: numberValue(scenario.gpu_utilization_maximum_percent, `${label}.gpu_utilization_maximum_percent`),
|
||||
gpuMemoryMaximumMib: numberValue(scenario.gpu_memory_maximum_mib, `${label}.gpu_memory_maximum_mib`),
|
||||
processPeakRssMib: numberValue(scenario.process_peak_rss_mib, `${label}.process_peak_rss_mib`),
|
||||
queueHighWatermarks: queues as M48SLoadEnvelopeScenario["queueHighWatermarks"],
|
||||
queueCapacity: integerValue(scenario.queue_capacity, `${label}.queue_capacity`),
|
||||
additionalInferencePasses: 0,
|
||||
repeatCount: 1,
|
||||
allRepetitionsPassed: targetGate,
|
||||
integrityGatePassed: true,
|
||||
operatingTargetGatePassed: targetGate,
|
||||
thresholds: {
|
||||
minimumDeliveryRatio: numberValue(thresholds.minimum_delivery_ratio, `${label}.thresholds.minimum_delivery_ratio`),
|
||||
minimumEffectiveWorldStateFps: numberValue(thresholds.minimum_effective_world_state_fps, `${label}.thresholds.minimum_effective_world_state_fps`),
|
||||
maximumWorldStateCompletionP95Ms: numberValue(thresholds.maximum_world_state_completion_p95_ms, `${label}.thresholds.maximum_world_state_completion_p95_ms`),
|
||||
},
|
||||
repetitions: [],
|
||||
};
|
||||
});
|
||||
if (scenarios.length !== 3) {
|
||||
throw new M48SFixedClassDetectorContractError("M4.8S.load_envelope.scenarios: неполный legacy-набор.");
|
||||
}
|
||||
return {
|
||||
repeatCountPerRate: 1,
|
||||
productionRateRepeatabilityPassed: false,
|
||||
reserve12FpsPassed: true,
|
||||
limit15FpsPassed: true,
|
||||
loadEnvelopeAccepted: false,
|
||||
computeCapacityAtLeastFps: 15,
|
||||
bottleneckInterpretation: "rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation",
|
||||
scenarios,
|
||||
};
|
||||
}
|
||||
exact(
|
||||
comparison.schema_version,
|
||||
"missioncore.m48s-load-envelope-comparison/v2",
|
||||
"M4.8S.metrics.load_envelope.schema_version",
|
||||
);
|
||||
exact(comparison.repeat_count_per_rate, 3, "M4.8S.load_envelope.repeat_count_per_rate");
|
||||
exact(comparison.production_rate_repeatability_passed, true, "M4.8S.load_envelope.production_rate_repeatability_passed");
|
||||
exact(comparison.reserve_12_fps_passed, true, "M4.8S.load_envelope.reserve_12_fps_passed");
|
||||
exact(comparison.limit_15_fps_passed, true, "M4.8S.load_envelope.limit_15_fps_passed");
|
||||
exact(comparison.load_envelope_accepted, true, "M4.8S.load_envelope.load_envelope_accepted");
|
||||
exact(comparison.compute_capacity_at_least_fps, 15, "M4.8S.load_envelope.compute_capacity_at_least_fps");
|
||||
exact(
|
||||
comparison.bottleneck_interpretation,
|
||||
"cold-video-decode-isolated-before-admission-no-steady-gpu-saturation",
|
||||
"M4.8S.load_envelope.bottleneck_interpretation",
|
||||
);
|
||||
const ids = ["production-10fps", "reserve-12fps", "limit-15fps"] as const;
|
||||
const purposes = ["production-rate", "reserve-gate", "limit-discovery"] as const;
|
||||
const scenarios = arrayValue(comparison.scenarios, "M4.8S.load_envelope.scenarios").map((value, index) => {
|
||||
const label = `M4.8S.load_envelope.scenarios[${index}]`;
|
||||
const scenario = objectValue(value, label);
|
||||
const id = ids[index];
|
||||
const purpose = purposes[index];
|
||||
if (id === undefined || purpose === undefined) {
|
||||
throw new M48SFixedClassDetectorContractError("M4.8S.load_envelope.scenarios: лишний сценарий.");
|
||||
}
|
||||
exact(scenario.id, id, `${label}.id`);
|
||||
exact(scenario.load_purpose, purpose, `${label}.load_purpose`);
|
||||
exact(scenario.repeat_count, 3, `${label}.repeat_count`);
|
||||
exact(scenario.all_repetitions_passed, true, `${label}.all_repetitions_passed`);
|
||||
exact(scenario.integrity_gate_passed, true, `${label}.integrity_gate_passed`);
|
||||
exact(scenario.operating_target_gate_passed, true, `${label}.operating_target_gate_passed`);
|
||||
exact(scenario.additional_inference_passes, 0, `${label}.additional_inference_passes`);
|
||||
const queues = integerRecordValue(scenario.queue_high_watermarks, `${label}.queue_high_watermarks`);
|
||||
for (const key of ["detector", "geometry", "temporal", "rolling", "threat"] as const) {
|
||||
if (!(key in queues)) {
|
||||
throw new M48SFixedClassDetectorContractError(`${label}.queue_high_watermarks.${key}: отсутствует.`);
|
||||
}
|
||||
}
|
||||
const thresholds = objectValue(scenario.thresholds, `${label}.thresholds`);
|
||||
const repetitions = arrayValue(scenario.repetitions, `${label}.repetitions`).map((value, repetitionIndex) => {
|
||||
const repetitionLabel = `${label}.repetitions[${repetitionIndex}]`;
|
||||
const repetition = objectValue(value, repetitionLabel);
|
||||
const repetitionNumber = repetitionIndex + 1;
|
||||
exact(repetition.repetition, repetitionNumber, `${repetitionLabel}.repetition`);
|
||||
exact(repetition.operating_target_gate_passed, true, `${repetitionLabel}.operating_target_gate_passed`);
|
||||
const runId = textValue(repetition.run_id, `${repetitionLabel}.run_id`);
|
||||
const resultSha256 = textValue(repetition.result_sha256, `${repetitionLabel}.result_sha256`);
|
||||
const frameEvidenceSha256 = textValue(repetition.frame_evidence_sha256, `${repetitionLabel}.frame_evidence_sha256`);
|
||||
if (!SHA256.test(resultSha256) || !SHA256.test(frameEvidenceSha256)) {
|
||||
throw new M48SFixedClassDetectorContractError(`${repetitionLabel}: нарушен SHA-256.`);
|
||||
}
|
||||
return {
|
||||
runId,
|
||||
repetition: repetitionNumber as 1 | 2 | 3,
|
||||
deliveredWorldStates: integerValue(repetition.delivered_world_states, `${repetitionLabel}.delivered_world_states`),
|
||||
supersededFrames: integerValue(repetition.superseded_frames, `${repetitionLabel}.superseded_frames`),
|
||||
deliveryRatio: numberValue(repetition.delivery_ratio, `${repetitionLabel}.delivery_ratio`),
|
||||
effectiveWorldStateFps: numberValue(repetition.effective_world_state_fps, `${repetitionLabel}.effective_world_state_fps`),
|
||||
worldStateCompletionAgeP95Ms: numberValue(repetition.world_state_completion_age_p95_ms, `${repetitionLabel}.world_state_completion_age_p95_ms`),
|
||||
hotLoopDecodeP95Ms: numberValue(repetition.hot_loop_decode_p95_ms, `${repetitionLabel}.hot_loop_decode_p95_ms`),
|
||||
hotLoopDecodeMaximumMs: numberValue(repetition.hot_loop_decode_maximum_ms, `${repetitionLabel}.hot_loop_decode_maximum_ms`),
|
||||
preadmissionDecodeMaximumMs: numberValue(repetition.preadmission_decode_maximum_ms, `${repetitionLabel}.preadmission_decode_maximum_ms`),
|
||||
sourcePacingLatenessMaximumMs: numberValue(repetition.source_pacing_lateness_maximum_ms, `${repetitionLabel}.source_pacing_lateness_maximum_ms`),
|
||||
operatingTargetGatePassed: true as const,
|
||||
resultSha256,
|
||||
frameEvidenceSha256,
|
||||
};
|
||||
});
|
||||
if (repetitions.length !== 3) {
|
||||
throw new M48SFixedClassDetectorContractError(`${label}.repetitions: требуется три повтора.`);
|
||||
}
|
||||
return {
|
||||
id,
|
||||
loadPurpose: purpose,
|
||||
requestedSourceRateHz: numberValue(scenario.requested_source_rate_hz, `${label}.requested_source_rate_hz`),
|
||||
sourceFramesAdmitted: integerValue(scenario.source_frames_admitted, `${label}.source_frames_admitted`),
|
||||
deliveredWorldStates: integerValue(scenario.delivered_world_states, `${label}.delivered_world_states`),
|
||||
supersededFrames: integerValue(scenario.superseded_frames, `${label}.superseded_frames`),
|
||||
deliveryRatio: numberValue(scenario.delivery_ratio, `${label}.delivery_ratio`),
|
||||
effectiveWorldStateFps: numberValue(scenario.effective_world_state_fps, `${label}.effective_world_state_fps`),
|
||||
worldStateCompletionAgeP95Ms: numberValue(scenario.world_state_completion_age_p95_ms, `${label}.world_state_completion_age_p95_ms`),
|
||||
worldStateCompletionAgeP99Ms: numberValue(scenario.world_state_completion_age_p99_ms, `${label}.world_state_completion_age_p99_ms`),
|
||||
worldStateCompletionAgeMaximumMs: numberValue(scenario.world_state_completion_age_maximum_ms, `${label}.world_state_completion_age_maximum_ms`),
|
||||
hotLoopDecodeP95Ms: numberValue(scenario.hot_loop_decode_p95_ms, `${label}.hot_loop_decode_p95_ms`),
|
||||
hotLoopDecodeMaximumMs: numberValue(scenario.hot_loop_decode_maximum_ms, `${label}.hot_loop_decode_maximum_ms`),
|
||||
preadmissionDecodeMaximumMs: numberValue(scenario.preadmission_decode_maximum_ms, `${label}.preadmission_decode_maximum_ms`),
|
||||
sourcePrefetchPreparationMaximumMs: numberValue(scenario.source_prefetch_preparation_maximum_ms, `${label}.source_prefetch_preparation_maximum_ms`),
|
||||
sourcePacingLatenessP95Ms: numberValue(scenario.source_pacing_lateness_p95_ms, `${label}.source_pacing_lateness_p95_ms`),
|
||||
sourcePacingLatenessMaximumMs: numberValue(scenario.source_pacing_lateness_maximum_ms, `${label}.source_pacing_lateness_maximum_ms`),
|
||||
detectorP95Ms: numberValue(scenario.detector_p95_ms, `${label}.detector_p95_ms`),
|
||||
detectorMaximumMs: numberValue(scenario.detector_maximum_ms, `${label}.detector_maximum_ms`),
|
||||
gpuUtilizationMeanPercent: numberValue(scenario.gpu_utilization_mean_percent, `${label}.gpu_utilization_mean_percent`),
|
||||
gpuUtilizationMaximumPercent: numberValue(scenario.gpu_utilization_maximum_percent, `${label}.gpu_utilization_maximum_percent`),
|
||||
gpuMemoryMaximumMib: numberValue(scenario.gpu_memory_maximum_mib, `${label}.gpu_memory_maximum_mib`),
|
||||
processPeakRssMib: numberValue(scenario.process_peak_rss_mib, `${label}.process_peak_rss_mib`),
|
||||
queueHighWatermarks: queues as M48SLoadEnvelopeScenario["queueHighWatermarks"],
|
||||
queueCapacity: integerValue(scenario.queue_capacity, `${label}.queue_capacity`),
|
||||
additionalInferencePasses: 0,
|
||||
repeatCount: 3 as const,
|
||||
allRepetitionsPassed: true as const,
|
||||
integrityGatePassed: true as const,
|
||||
operatingTargetGatePassed: true as const,
|
||||
thresholds: {
|
||||
minimumDeliveryRatio: numberValue(thresholds.minimum_delivery_ratio, `${label}.thresholds.minimum_delivery_ratio`),
|
||||
minimumEffectiveWorldStateFps: numberValue(thresholds.minimum_effective_world_state_fps, `${label}.thresholds.minimum_effective_world_state_fps`),
|
||||
maximumWorldStateCompletionP95Ms: numberValue(thresholds.maximum_world_state_completion_p95_ms, `${label}.thresholds.maximum_world_state_completion_p95_ms`),
|
||||
},
|
||||
repetitions,
|
||||
};
|
||||
});
|
||||
if (scenarios.length !== ids.length) {
|
||||
throw new M48SFixedClassDetectorContractError("M4.8S.load_envelope.scenarios: неполный набор.");
|
||||
}
|
||||
return {
|
||||
repeatCountPerRate: 3,
|
||||
productionRateRepeatabilityPassed: true,
|
||||
reserve12FpsPassed: true,
|
||||
limit15FpsPassed: true,
|
||||
loadEnvelopeAccepted: true,
|
||||
computeCapacityAtLeastFps: 15,
|
||||
bottleneckInterpretation: "cold-video-decode-isolated-before-admission-no-steady-gpu-saturation",
|
||||
scenarios,
|
||||
};
|
||||
}
|
||||
|
||||
function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorResult {
|
||||
const payload = objectValue(value, "M4.8S");
|
||||
exact(
|
||||
@@ -384,6 +768,9 @@ function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorRe
|
||||
const metrics = objectValue(payload.metrics, "M4.8S.metrics");
|
||||
const load = objectValue(metrics.detector_load, "M4.8S.metrics.detector_load");
|
||||
const decision = objectValue(payload.decision, "M4.8S.decision");
|
||||
const loadEnvelope = metrics.load_envelope === undefined
|
||||
? null
|
||||
: loadEnvelopeComparisonValue(metrics.load_envelope);
|
||||
const cameraRaster = arrayValue(source.camera_raster, "M4.8S.source.camera_raster");
|
||||
if (cameraRaster.length !== 2) {
|
||||
throw new M48SFixedClassDetectorContractError("M4.8S.source.camera_raster: нарушен размер.");
|
||||
@@ -412,6 +799,13 @@ function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorRe
|
||||
exact(decision.detector_replacement_authorized, false, "M4.8S.decision.detector_replacement_authorized");
|
||||
}
|
||||
exact(decision.production_accepted, false, "M4.8S.decision.production_accepted");
|
||||
if (loadEnvelope !== null) {
|
||||
exact(decision.load_envelope_evaluated, true, "M4.8S.decision.load_envelope_evaluated");
|
||||
exact(decision.production_rate_repeatability_passed, loadEnvelope.productionRateRepeatabilityPassed, "M4.8S.decision.production_rate_repeatability_passed");
|
||||
exact(decision.reserve_12_fps_passed, loadEnvelope.reserve12FpsPassed, "M4.8S.decision.reserve_12_fps_passed");
|
||||
exact(decision.limit_15_fps_passed, loadEnvelope.limit15FpsPassed, "M4.8S.decision.limit_15_fps_passed");
|
||||
exact(decision.load_envelope_accepted, loadEnvelope.loadEnvelopeAccepted, "M4.8S.decision.load_envelope_accepted");
|
||||
}
|
||||
const frames = arrayValue(payload.frames, "M4.8S.frames").map(descriptorValue);
|
||||
const evidenceFrameCount = integerValue(source.evidence_frame_count, "M4.8S.source.evidence_frame_count");
|
||||
if (frames.length !== evidenceFrameCount || new Set(frames.map((frame) => frame.frameId)).size !== frames.length) {
|
||||
@@ -457,6 +851,10 @@ function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorRe
|
||||
integratedWorldState: integratedGate
|
||||
? integratedWorldStateValue(metrics.integrated_world_state)
|
||||
: null,
|
||||
runtimeHardening: metrics.runtime_hardening === undefined
|
||||
? null
|
||||
: runtimeHardeningComparisonValue(metrics.runtime_hardening),
|
||||
loadEnvelope,
|
||||
},
|
||||
decision: {
|
||||
selectedCandidate: "rf-detr",
|
||||
@@ -464,6 +862,11 @@ function parseResult(value: unknown, resultId: string): M48SFixedClassDetectorRe
|
||||
integratedWorldStateGateEvaluated: integratedGate,
|
||||
integratedWorldStateGatePassed: integratedGate,
|
||||
detectorReplacementAuthorized: false,
|
||||
loadEnvelopeEvaluated: loadEnvelope !== null,
|
||||
productionRateRepeatabilityPassed: loadEnvelope?.productionRateRepeatabilityPassed ?? false,
|
||||
reserve12FpsPassed: loadEnvelope?.reserve12FpsPassed ?? false,
|
||||
limit15FpsPassed: loadEnvelope?.limit15FpsPassed ?? false,
|
||||
loadEnvelopeAccepted: loadEnvelope?.loadEnvelopeAccepted ?? false,
|
||||
productionAccepted: false,
|
||||
},
|
||||
limitations: arrayValue(payload.limitations, "M4.8S.limitations").map((item, index) => textValue(item, `M4.8S.limitations[${index}]`)),
|
||||
|
||||
@@ -19,9 +19,24 @@ export function M48SFixedClassDetectorResultView({
|
||||
result: M48SFixedClassDetectorResult;
|
||||
}) {
|
||||
const selected = result.metrics.candidates.find((candidate) => candidate.selected);
|
||||
const load = result.metrics.detectorLoad;
|
||||
const detectorLoad = result.metrics.detectorLoad;
|
||||
const integrated = result.metrics.integratedWorldState;
|
||||
const status = integrated
|
||||
const hardening = result.metrics.runtimeHardening;
|
||||
const loadEnvelope = result.metrics.loadEnvelope;
|
||||
const production = loadEnvelope?.scenarios.find((scenario) => scenario.id === "production-10fps");
|
||||
const reserve = loadEnvelope?.scenarios.find((scenario) => scenario.id === "reserve-12fps");
|
||||
const limit = loadEnvelope?.scenarios.find((scenario) => scenario.id === "limit-15fps");
|
||||
const load = loadEnvelope && production && reserve && limit
|
||||
? { comparison: loadEnvelope, production, reserve, limit }
|
||||
: null;
|
||||
const loadAccepted = load?.comparison.loadEnvelopeAccepted === true;
|
||||
const status = load
|
||||
? loadAccepted
|
||||
? "3 × 10 / 12 / 15 FPS прошли; realtime envelope ≥ 15 FPS принят"
|
||||
: "Capacity ≥ 15 FPS подтверждён; repeatability 10 FPS требует закрытия"
|
||||
: hardening
|
||||
? `После hardening: ${hardening.hardened.deliveredWorldStates.toLocaleString("ru-RU")} из ${hardening.hardened.sourceFramesAdmitted.toLocaleString("ru-RU")} world states доставлены`
|
||||
: integrated
|
||||
? "Полный RF-DETR reference graph выдержал realtime shadow"
|
||||
: "RF-DETR-L выдержал detector-only realtime shadow";
|
||||
return (
|
||||
@@ -29,9 +44,11 @@ export function M48SFixedClassDetectorResultView({
|
||||
summary={(
|
||||
<LaboratorySummary
|
||||
title="M4.8S · fixed-class semantics риск-объектов"
|
||||
description="Сравнение трёх готовых COCO-детекторов на точных кадрах RAVNOVES00, 30-минутная квалификация RF-DETR-L и полный source-paced прогон RF-DETR → geometry → temporal → motion → rolling map → threat на Worker 006. Статические препятствия остаются в геометрическом контуре; классы используются только там, где меняется ожидаемое поведение."
|
||||
description={loadAccepted
|
||||
? "RF-DETR-L встроен в полный source-paced граф RF-DETR → geometry → temporal → motion → rolling map → threat. Холодный video decode вынесен до допуска источника, после чего один immutable проход выполнен по три раза на 10, 12 и 15 FPS. LAB показывает худший результат каждой серии, хвосты decode/pacing и каждый из девяти sealed-прогонов."
|
||||
: "RF-DETR-L встроен в полный source-paced граф RF-DETR → geometry → temporal → motion → rolling map → threat. Исторический immutable проход отдельно измерен на 10, 12 и 15 FPS; незакрытый 10 FPS delivery gate сохранён без постфактум-сдвига thresholds."}
|
||||
status={status}
|
||||
statusTone="success"
|
||||
statusTone={load && !loadAccepted ? "warning" : "success"}
|
||||
facts={[
|
||||
{ label: "Источник", value: `${rigLabel} RIGHT · raw KB4 · ${result.source.evidenceFrameCount} diagnostic frames` },
|
||||
{ label: "Сравнение", value: "YOLOX-S · D-FINE-S · RF-DETR-L · единый threshold 0.50" },
|
||||
@@ -39,12 +56,28 @@ export function M48SFixedClassDetectorResultView({
|
||||
{ label: "Authority", value: "SHADOW ONLY · commands OFF · actuation OFF · production NO" },
|
||||
]}
|
||||
brief={{
|
||||
question: "Можно ли заменить слабую class-семантику YOLOX готовой моделью, не потеряв realtime на предельном Worker с RTX 4090?",
|
||||
approach: `YOLOX-S, D-FINE-S и RF-DETR-L сравнили на одинаковых ${result.source.evidenceFrameCount} raw-KB4 кадрах с порогом 0.50. RF-DETR-L отдельно квалифицировали ${decimal(load.durationSeconds / 60, 0)} минут, затем встроили в полный reference graph без дополнительного inference-прохода.`,
|
||||
principalResult: integrated
|
||||
question: load
|
||||
? "Какой realtime-запас имеет неизменный single-pass RF-DETR graph на Worker 006 и где начинается его вычислительный предел?"
|
||||
: "Можно ли заменить слабую class-семантику YOLOX готовой моделью, не потеряв realtime на предельном Worker с RTX 4090?",
|
||||
approach: loadAccepted
|
||||
? "Один и тот же проход из 4489 кадров прогнали строго последовательно девять раз: по три повтора на 10, 12 и 15 FPS. Bounded prefetch на 64 кадра завершался до source admission; sensor timestamps, модель, providers, очереди и single inference не менялись. Пороги и порядок были записаны заранее."
|
||||
: load
|
||||
? "Один и тот же проход из 4489 кадров прогнали последовательно на 10, 12 и 15 FPS. Sensor timestamps, модель, graph providers, очереди, prewarm и single inference не менялись; менялся только wall-clock pacing. Пороги delivery/FPS/p95 были записаны до запусков."
|
||||
: `YOLOX-S, D-FINE-S и RF-DETR-L сравнили на одинаковых ${result.source.evidenceFrameCount} raw-KB4 кадрах с порогом 0.50. RF-DETR-L отдельно квалифицировали ${decimal(result.metrics.detectorLoad.durationSeconds / 60, 0)} минут, затем встроили в полный reference graph без дополнительного inference-прохода.`,
|
||||
principalResult: loadAccepted && load
|
||||
? `Все девять прогонов прошли integrity и predeclared target gates. Худший результат: 10 FPS — ${load.production.deliveredWorldStates.toLocaleString("ru-RU")}/${load.production.sourceFramesAdmitted.toLocaleString("ru-RU")}, 12 FPS — ${load.reserve.deliveredWorldStates.toLocaleString("ru-RU")}/${load.reserve.sourceFramesAdmitted.toLocaleString("ru-RU")}, 15 FPS — ${load.limit.deliveredWorldStates.toLocaleString("ru-RU")}/${load.limit.sourceFramesAdmitted.toLocaleString("ru-RU")}. Во всех сериях сохранён один inference на кадр.`
|
||||
: load
|
||||
? `12 FPS прошли reserve-gate: ${load.reserve.deliveredWorldStates.toLocaleString("ru-RU")}/${load.reserve.sourceFramesAdmitted.toLocaleString("ru-RU")} доставлено. 15 FPS также прошли: ${load.limit.deliveredWorldStates.toLocaleString("ru-RU")}/${load.limit.sourceFramesAdmitted.toLocaleString("ru-RU")}. Но 10 FPS не добрали заранее заданный delivery ratio: ${load.production.deliveredWorldStates.toLocaleString("ru-RU")}/${load.production.sourceFramesAdmitted.toLocaleString("ru-RU")}.`
|
||||
: hardening
|
||||
? `На одинаковых ${hardening.baseline.sourceFramesAdmitted.toLocaleString("ru-RU")} входных кадрах полный граф увеличил доставку с ${hardening.baseline.deliveredWorldStates.toLocaleString("ru-RU")} до ${hardening.hardened.deliveredWorldStates.toLocaleString("ru-RU")} world states, а число вытесненных кадров снизил с ${hardening.baseline.supersededFrames} до ${hardening.hardened.supersededFrames}. Отдельный ${hardening.startup.validationFrames.toLocaleString("ru-RU")}-кадровый прогон подтвердил prewarm до допуска источника.`
|
||||
: integrated
|
||||
? `Полный граф доставил ${integrated.deliveredWorldStates.toLocaleString("ru-RU")} world states при ${decimal(integrated.effectiveWorldStateFps, 3)} FPS и p95 ${decimal(integrated.worldStateCompletionAgeP95Ms, 3)} ms; ${integrated.supersededFrames} входных кадров штатно вытеснены latest-wins очередью.`
|
||||
: `RF-DETR-L выбран из трёх кандидатов и обработал ${load.sourceFramesConsumed.toLocaleString("ru-RU")} кадров detector-only без замен и ошибок.`,
|
||||
limitation: "Прогон доказывает runtime envelope, а не истинность классов, качество track identity, корректность risk policy или безопасность движения. Production authority и команды отключены.",
|
||||
: `RF-DETR-L выбран из трёх кандидатов и обработал ${detectorLoad.sourceFramesConsumed.toLocaleString("ru-RU")} кадров detector-only без замен и ошибок.`,
|
||||
limitation: loadAccepted
|
||||
? "Runtime repeatability и capacity полного графа закрыты как минимум до 15 FPS. Это не доказывает истинность классов, track/risk truth, collision safety или physical-live поведение; команды и production authority по-прежнему отключены."
|
||||
: load
|
||||
? "Capacity полного графа доказан как минимум до 15 FPS, но исторический production repeatability gate не принят. Истинность классов, collision safety и physical-live не проверялись; команды отключены."
|
||||
: "Прогон доказывает runtime envelope, а не истинность классов, качество track identity, корректность risk policy или безопасность движения. Production authority и команды отключены.",
|
||||
}}
|
||||
method={{
|
||||
completeness: result.method.completeness,
|
||||
@@ -73,28 +106,67 @@ export function M48SFixedClassDetectorResultView({
|
||||
)}
|
||||
result={(
|
||||
<LaboratoryResultSummary
|
||||
title={integrated
|
||||
title={hardening
|
||||
? load
|
||||
? "Что сравнивать: один graph на 10 / 12 / 15 FPS"
|
||||
: "Что сравнивать: один и тот же полный прогон до и после hardening"
|
||||
: integrated
|
||||
? "Что дал прогон: полный world-state graph проходит realtime envelope"
|
||||
: "Что дал прогон: RF-DETR-L проходит detector-only realtime envelope"}
|
||||
status={status}
|
||||
statusTone="success"
|
||||
metrics={integrated ? [
|
||||
statusTone={load && !loadAccepted ? "warning" : "success"}
|
||||
metrics={load ? [
|
||||
{ label: "Effective world-state FPS", value: `${decimal(load.production.effectiveWorldStateFps, 3)} / ${decimal(load.reserve.effectiveWorldStateFps, 3)} / ${decimal(load.limit.effectiveWorldStateFps, 3)}`, hint: "вход 10 / 12 / 15 FPS" },
|
||||
{ label: "Worst delivered / superseded", value: `${load.production.deliveredWorldStates.toLocaleString("ru-RU")}/${load.production.supersededFrames} · ${load.reserve.deliveredWorldStates.toLocaleString("ru-RU")}/${load.reserve.supersededFrames} · ${load.limit.deliveredWorldStates.toLocaleString("ru-RU")}/${load.limit.supersededFrames}`, hint: "минимальная доставка / максимум вытеснений из трёх повторов" },
|
||||
{ label: "World-state p95", value: `${decimal(load.production.worldStateCompletionAgeP95Ms, 3)} / ${decimal(load.reserve.worldStateCompletionAgeP95Ms, 3)} / ${decimal(load.limit.worldStateCompletionAgeP95Ms, 3)} ms`, hint: "10 / 12 / 15 FPS" },
|
||||
{ label: "World-state p99", value: `${decimal(load.production.worldStateCompletionAgeP99Ms, 3)} / ${decimal(load.reserve.worldStateCompletionAgeP99Ms, 3)} / ${decimal(load.limit.worldStateCompletionAgeP99Ms, 3)} ms`, hint: "хвост не растёт вместе с rate" },
|
||||
{ label: loadAccepted ? "Hot-loop decode p95" : "Decode p95", value: `${decimal(load.production.hotLoopDecodeP95Ms, 3)} / ${decimal(load.reserve.hotLoopDecodeP95Ms, 3)} / ${decimal(load.limit.hotLoopDecodeP95Ms, 3)} ms`, hint: loadAccepted ? "холодный open/decode исключён из realtime" : "исторический decode включает cold tail" },
|
||||
loadAccepted
|
||||
? { label: "Prefetch / pacing max", value: `${decimal(Math.max(...load.comparison.scenarios.map((scenario) => scenario.sourcePrefetchPreparationMaximumMs)) / 1000, 2)} s / ${decimal(Math.max(...load.comparison.scenarios.map((scenario) => scenario.sourcePacingLatenessMaximumMs)), 3)} ms`, hint: "prefetch до admission / wall-clock pacing в hot loop" }
|
||||
: { label: "Decode maximum", value: `${decimal(load.production.hotLoopDecodeMaximumMs, 0)} / ${decimal(load.reserve.hotLoopDecodeMaximumMs, 0)} / ${decimal(load.limit.hotLoopDecodeMaximumMs, 0)} ms`, hint: "редкий cold source/decode tail" },
|
||||
{ label: "Detector p95", value: `${decimal(load.production.detectorP95Ms, 3)} / ${decimal(load.reserve.detectorP95Ms, 3)} / ${decimal(load.limit.detectorP95Ms, 3)} ms`, hint: "тот же RF-DETR · один inference" },
|
||||
{ label: "GPU mean", value: `${decimal(load.production.gpuUtilizationMeanPercent)}% / ${decimal(load.reserve.gpuUtilizationMeanPercent)}% / ${decimal(load.limit.gpuUtilizationMeanPercent)}%`, hint: "10 / 12 / 15 FPS · насыщения нет" },
|
||||
{ label: "VRAM / process RAM peak", value: `${decimal(Math.max(...load.comparison.scenarios.map((scenario) => scenario.gpuMemoryMaximumMib)) / 1024, 2)} / ${decimal(Math.max(...load.comparison.scenarios.map((scenario) => scenario.processPeakRssMib)) / 1024, 2)} GiB`, hint: "общий GPU counter / graph process RSS" },
|
||||
] : hardening ? [
|
||||
{ label: "World-state FPS", value: `${decimal(hardening.baseline.effectiveWorldStateFps, 3)} → ${decimal(hardening.hardened.effectiveWorldStateFps, 3)}`, hint: "до → после · target ≥ 9.5 FPS" },
|
||||
{ label: "Задержка p95", value: `${decimal(hardening.baseline.worldStateCompletionAgeP95Ms, 3)} → ${decimal(hardening.hardened.worldStateCompletionAgeP95Ms, 3)} ms`, hint: `p99 ${decimal(hardening.baseline.worldStateCompletionAgeP99Ms, 3)} → ${decimal(hardening.hardened.worldStateCompletionAgeP99Ms, 3)} ms` },
|
||||
{ label: "Максимальная задержка", value: `${decimal(hardening.baseline.worldStateCompletionAgeMaximumMs, 3)} → ${decimal(hardening.hardened.worldStateCompletionAgeMaximumMs, 3)} ms`, hint: "полный прогон без prewarm" },
|
||||
{ label: "Доставлено / вытеснено", value: `${hardening.baseline.deliveredWorldStates.toLocaleString("ru-RU")}/${hardening.baseline.supersededFrames} → ${hardening.hardened.deliveredWorldStates.toLocaleString("ru-RU")}/${hardening.hardened.supersededFrames}`, hint: `${hardening.hardened.sourceFramesAdmitted.toLocaleString("ru-RU")} одинаковых входных кадров` },
|
||||
{ label: "Rolling max", value: `${decimal(hardening.baseline.rollingMaximumMs, 3)} → ${decimal(hardening.hardened.rollingMaximumMs, 3)} ms`, hint: "убран длинный стоп realtime-контура" },
|
||||
{ label: "Geometry max", value: `${decimal(hardening.baseline.geometryMaximumMs, 3)} → ${decimal(hardening.hardened.geometryMaximumMs, 3)} ms`, hint: "тот же полный source-paced прогон" },
|
||||
{ label: "Первый detector frame", value: `${decimal(hardening.startup.baseline.detectorMs, 3)} → ${decimal(hardening.startup.prewarmed.detectorMs, 3)} ms`, hint: `prewarm ${decimal(hardening.startup.prewarmDurationMs, 3)} ms до допуска source` },
|
||||
{ label: "Первый world state", value: `${decimal(hardening.startup.baseline.worldStateMs, 3)} → ${decimal(hardening.startup.prewarmed.worldStateMs, 3)} ms`, hint: `${hardening.startup.validationFrames.toLocaleString("ru-RU")} кадров · ${hardening.startup.prewarmInferencePasses} warmup inference` },
|
||||
] : integrated ? [
|
||||
{ label: "Complete graph", value: `${decimal(integrated.effectiveWorldStateFps, 3)} FPS`, hint: "target ≥ 9.5 FPS · source-paced" },
|
||||
{ label: "World-state age p95", value: `${decimal(integrated.worldStateCompletionAgeP95Ms, 3)} ms`, hint: `p99 ${decimal(integrated.worldStateCompletionAgeP99Ms, 3)} ms · target ≤ 175 ms` },
|
||||
{ label: "Delivered / superseded", value: `${integrated.deliveredWorldStates.toLocaleString("ru-RU")} / ${integrated.supersededFrames}`, hint: `${integrated.failures} failures · queues ${Math.max(...Object.values(integrated.queueHighWatermarks))}/${integrated.queueCapacity}` },
|
||||
{ label: "GPU / VRAM peak", value: `${decimal(integrated.gpuUtilizationMaximumPercent, 0)}% / ${decimal(integrated.gpuMemoryMaximumMib / 1024)} GiB`, hint: `GPU mean ${decimal(integrated.gpuUtilizationMeanPercent)}% · ${decimal(integrated.gpuPowerMaximumW)} W` },
|
||||
] : [
|
||||
{ label: "Detector capacity", value: `${decimal(selected?.capacityFps ?? 0)} FPS`, hint: "RF-DETR-L TensorRT/Triton" },
|
||||
{ label: "Completion age p95", value: `${decimal(load.completionAgeP95Ms)} ms`, hint: "detector-only · target ≤ 175 ms" },
|
||||
{ label: "Consumed / replaced", value: `${load.sourceFramesConsumed.toLocaleString("ru-RU")} / ${load.sourceFrameReplacements}`, hint: `${decimal(load.effectiveConsumedFps, 3)} source FPS · ${load.failures} failures` },
|
||||
{ label: "GPU / VRAM peak", value: `${decimal(load.gpuUtilizationMaximumPercent, 0)}% / ${decimal(load.gpuMemoryMaximumMib / 1024)} GiB`, hint: `GPU mean ${decimal(load.gpuUtilizationMeanPercent)}% · queue ${load.queueMaximumDepth}/${load.queueCapacity}` },
|
||||
{ label: "Completion age p95", value: `${decimal(detectorLoad.completionAgeP95Ms)} ms`, hint: "detector-only · target ≤ 175 ms" },
|
||||
{ label: "Consumed / replaced", value: `${detectorLoad.sourceFramesConsumed.toLocaleString("ru-RU")} / ${detectorLoad.sourceFrameReplacements}`, hint: `${decimal(detectorLoad.effectiveConsumedFps, 3)} source FPS · ${detectorLoad.failures} failures` },
|
||||
{ label: "GPU / VRAM peak", value: `${decimal(detectorLoad.gpuUtilizationMaximumPercent, 0)}% / ${decimal(detectorLoad.gpuMemoryMaximumMib / 1024)} GiB`, hint: `GPU mean ${decimal(detectorLoad.gpuUtilizationMeanPercent)}% · queue ${detectorLoad.queueMaximumDepth}/${detectorLoad.queueCapacity}` },
|
||||
]}
|
||||
conclusion={{
|
||||
proved: integrated
|
||||
proved: load
|
||||
? `Все ${load.comparison.repeatCountPerRate * load.comparison.scenarios.length} полных single-pass прогона сохранили bounded queues 2/2, нулевые failed/stale/rejected/unavailable и нулевые дополнительные inference-проходы. Худший 10 FPS delivery — ${(load.production.deliveryRatio * 100).toLocaleString("ru-RU", { maximumFractionDigits: 3 })}%, 12 FPS — ${(load.reserve.deliveryRatio * 100).toLocaleString("ru-RU", { maximumFractionDigits: 3 })}%, 15 FPS — ${(load.limit.deliveryRatio * 100).toLocaleString("ru-RU", { maximumFractionDigits: 3 })}%. Realtime capacity доказана как минимум до 15 FPS без более мощного Worker.`
|
||||
: hardening
|
||||
? `На полном сравнительном прогоне с ${hardening.hardened.sourceFramesAdmitted.toLocaleString("ru-RU")} одинаковыми входными кадрами hardening сохранил realtime FPS, доставил на ${(hardening.hardened.deliveredWorldStates - hardening.baseline.deliveredWorldStates).toLocaleString("ru-RU")} world states больше и сократил вытеснения ${hardening.baseline.supersededFrames} → ${hardening.hardened.supersededFrames}. Максимальный стоп rolling stage уменьшился ${decimal(hardening.baseline.rollingMaximumMs, 3)} → ${decimal(hardening.hardened.rollingMaximumMs, 3)} ms. Отдельный prewarm-прогон снизил первый detector frame ${decimal(hardening.startup.baseline.detectorMs, 3)} → ${decimal(hardening.startup.prewarmed.detectorMs, 3)} ms и первый world state ${decimal(hardening.startup.baseline.worldStateMs, 3)} → ${decimal(hardening.startup.prewarmed.worldStateMs, 3)} ms без дополнительного inference на кадрах.`
|
||||
: integrated
|
||||
? `На Worker 006 полный граф обработал ${integrated.sourceFramesAdmitted.toLocaleString("ru-RU")} входных кадров, доставил ${integrated.deliveredWorldStates.toLocaleString("ru-RU")} состояний без ошибок, удержал все очереди в пределах ${integrated.queueCapacity} и p95 ${decimal(integrated.worldStateCompletionAgeP95Ms, 3)} ms. Advisory сформировал публикации по семействам: geometry-only (${integrated.advisoryFamilyCounts["generic-obstacle"].toLocaleString("ru-RU")}), люди (${integrated.advisoryFamilyCounts.person.toLocaleString("ru-RU")}), животные (${integrated.advisoryFamilyCounts.animal.toLocaleString("ru-RU")}) и транспорт (${integrated.advisoryFamilyCounts.vehicle.toLocaleString("ru-RU")}); это не количество уникальных физических объектов и не потребовало второго inference.`
|
||||
: `RF-DETR-L ${decimal(load.durationSeconds / 60, 0)} минут устойчиво потреблял source-paced поток около 10 FPS: ${load.sourceFramesConsumed.toLocaleString("ru-RU")} кадров, 0 замен, 0 ошибок, completion-age p95 ${decimal(load.completionAgeP95Ms, 3)} ms.`,
|
||||
notProved: "Не доказаны unbiased precision/recall классов, независимое качество track identity и risk policy, поведение planner или collision safety. Кадровые рамки не заменяют геометрическую occupancy-карту.",
|
||||
decision: "Сохранить RF-DETR-L как risk-semantic shadow provider полного reference graph. Не классифицировать миллионы статических форм: неизвестное неподвижное препятствие остаётся geometry-owned и объезжается; классы сохраняются для людей, животных и транспорта. Production switch не разрешён.",
|
||||
: `RF-DETR-L ${decimal(detectorLoad.durationSeconds / 60, 0)} минут устойчиво потреблял source-paced поток около 10 FPS: ${detectorLoad.sourceFramesConsumed.toLocaleString("ru-RU")} кадров, 0 замен, 0 ошибок, completion-age p95 ${decimal(detectorLoad.completionAgeP95Ms, 3)} ms.`,
|
||||
notProved: loadAccepted
|
||||
? "Не доказаны unbiased precision/recall классов, независимая истинность track identity и risk policy, работа planner, collision safety или physical-live режим. Принятый runtime envelope не включает production/navigation authority."
|
||||
: load
|
||||
? `Не закрыта repeatability delivery на production-rate: 10 FPS дали ${(load.production.deliveryRatio * 100).toLocaleString("ru-RU", { maximumFractionDigits: 3 })}% при gate ${(load.production.thresholds.minimumDeliveryRatio * 100).toLocaleString("ru-RU", { maximumFractionDigits: 1 })}%. Исторический v1 результат сохранён; он не переинтерпретируется после появления v2.`
|
||||
: "Не доказаны unbiased precision/recall классов, независимое качество track identity и risk policy, поведение planner или collision safety. Кадровые рамки не заменяют геометрическую occupancy-карту.",
|
||||
decision: loadAccepted
|
||||
? "Сохранить bounded 64-frame prefetch до source admission как runtime baseline. Не усиливать GPU и не добавлять второй детектор: Worker 006 уже держит как минимум 15 FPS. Следующий этап — semantic/track/risk truth и physical-live shadow; production switch не разрешён."
|
||||
: load
|
||||
? "Историческое решение v1: не усиливать GPU и не добавлять второй детектор; изолировать cold video decode и повторить sealed 10/12/15 FPS серией прогонов. Результат v2 закрывает этот следующий этап отдельно, не переписывая v1."
|
||||
: hardening
|
||||
? "Сохранить разгрузку cyclic GC и обязательный detector prewarm как текущий runtime baseline. Следующий этап — измерять запас realtime при росте сцены и числа семантических объектов, не добавляя второй детектор и не расширяя inference-нагрузку. Production switch не разрешён."
|
||||
: "Сохранить RF-DETR-L как risk-semantic shadow provider полного reference graph. Не классифицировать миллионы статических форм: неизвестное неподвижное препятствие остаётся geometry-owned и объезжается; классы сохраняются для людей, животных и транспорта. Production switch не разрешён.",
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
@@ -45,6 +45,59 @@ function resultPayload() {
|
||||
selected: false,
|
||||
...overrides,
|
||||
});
|
||||
const loadScenario = (id, loadPurpose, rate, delivered, superseded) => ({
|
||||
id,
|
||||
load_purpose: loadPurpose,
|
||||
requested_source_rate_hz: rate,
|
||||
source_frames_admitted: 4489,
|
||||
delivered_world_states: delivered,
|
||||
superseded_frames: superseded,
|
||||
delivery_ratio: delivered / 4489,
|
||||
effective_world_state_fps: rate * 0.975,
|
||||
world_state_completion_age_p95_ms: 75,
|
||||
world_state_completion_age_p99_ms: 100,
|
||||
world_state_completion_age_maximum_ms: 400,
|
||||
hot_loop_decode_p95_ms: 14,
|
||||
hot_loop_decode_maximum_ms: 42,
|
||||
preadmission_decode_maximum_ms: 3633,
|
||||
source_prefetch_preparation_maximum_ms: 4353,
|
||||
source_pacing_lateness_p95_ms: 1.7,
|
||||
source_pacing_lateness_maximum_ms: 11,
|
||||
detector_p95_ms: 40,
|
||||
detector_maximum_ms: 370,
|
||||
gpu_utilization_mean_percent: 35,
|
||||
gpu_utilization_maximum_percent: 85,
|
||||
gpu_memory_maximum_mib: 9584,
|
||||
process_peak_rss_mib: 2327,
|
||||
queue_high_watermarks: { detector: 2, geometry: 2, temporal: 2, rolling: 2, threat: 2 },
|
||||
queue_capacity: 2,
|
||||
additional_inference_passes: 0,
|
||||
repeat_count: 3,
|
||||
all_repetitions_passed: true,
|
||||
integrity_gate_passed: true,
|
||||
operating_target_gate_passed: true,
|
||||
thresholds: {
|
||||
minimum_delivery_ratio: rate === 10 ? 0.999 : rate === 12 ? 0.995 : 0.95,
|
||||
minimum_effective_world_state_fps: rate * 0.95,
|
||||
maximum_world_state_completion_p95_ms: rate === 10 ? 125 : rate === 12 ? 150 : 175,
|
||||
},
|
||||
repetitions: [1, 2, 3].map((repetition) => ({
|
||||
run_id: `m48s-load-prefetch-v1-${id}-a${repetition}`,
|
||||
repetition,
|
||||
delivered_world_states: delivered,
|
||||
superseded_frames: superseded,
|
||||
delivery_ratio: delivered / 4489,
|
||||
effective_world_state_fps: rate * 0.975,
|
||||
world_state_completion_age_p95_ms: 75,
|
||||
hot_loop_decode_p95_ms: 14,
|
||||
hot_loop_decode_maximum_ms: 42,
|
||||
preadmission_decode_maximum_ms: 3633,
|
||||
source_pacing_lateness_maximum_ms: 11,
|
||||
operating_target_gate_passed: true,
|
||||
result_sha256: "7".repeat(64),
|
||||
frame_evidence_sha256: "8".repeat(64),
|
||||
})),
|
||||
});
|
||||
return {
|
||||
schema_version: "missioncore.m48s-fixed-class-detector-result-view/v1",
|
||||
result_id: resultId,
|
||||
@@ -138,6 +191,55 @@ function resultPayload() {
|
||||
additional_inference_passes: 0,
|
||||
failures: 0,
|
||||
},
|
||||
runtime_hardening: {
|
||||
schema_version: "missioncore.m48s-runtime-hardening-comparison/v1",
|
||||
baseline: {
|
||||
source_frames_admitted: 4489,
|
||||
delivered_world_states: 4480,
|
||||
superseded_frames: 9,
|
||||
effective_world_state_fps: 9.750635,
|
||||
world_state_completion_age_p95_ms: 76.886564,
|
||||
world_state_completion_age_p99_ms: 100.661219,
|
||||
world_state_completion_age_maximum_ms: 794.748644,
|
||||
rolling_maximum_ms: 762.638263,
|
||||
geometry_maximum_ms: 350.066302,
|
||||
additional_inference_passes: 0,
|
||||
},
|
||||
hardened: {
|
||||
source_frames_admitted: 4489,
|
||||
delivered_world_states: 4488,
|
||||
superseded_frames: 1,
|
||||
effective_world_state_fps: 9.821942,
|
||||
world_state_completion_age_p95_ms: 75.270655,
|
||||
world_state_completion_age_p99_ms: 94.309605,
|
||||
world_state_completion_age_maximum_ms: 449.893287,
|
||||
rolling_maximum_ms: 28.044958,
|
||||
geometry_maximum_ms: 49.174149,
|
||||
additional_inference_passes: 0,
|
||||
},
|
||||
startup: {
|
||||
baseline: { detector_ms: 397.356888, world_state_ms: 449.893287 },
|
||||
prewarmed: { detector_ms: 27.662887, world_state_ms: 60.516957 },
|
||||
prewarm_duration_ms: 420.721918,
|
||||
prewarm_inference_passes: 1,
|
||||
validation_frames: 1000,
|
||||
},
|
||||
},
|
||||
load_envelope: {
|
||||
schema_version: "missioncore.m48s-load-envelope-comparison/v2",
|
||||
repeat_count_per_rate: 3,
|
||||
production_rate_repeatability_passed: true,
|
||||
reserve_12_fps_passed: true,
|
||||
limit_15_fps_passed: true,
|
||||
load_envelope_accepted: true,
|
||||
compute_capacity_at_least_fps: 15,
|
||||
bottleneck_interpretation: "cold-video-decode-isolated-before-admission-no-steady-gpu-saturation",
|
||||
scenarios: [
|
||||
loadScenario("production-10fps", "production-rate", 10, 4489, 0),
|
||||
loadScenario("reserve-12fps", "reserve-gate", 12, 4488, 1),
|
||||
loadScenario("limit-15fps", "limit-discovery", 15, 4488, 1),
|
||||
],
|
||||
},
|
||||
},
|
||||
decision: {
|
||||
selected_candidate: "rf-detr",
|
||||
@@ -145,6 +247,11 @@ function resultPayload() {
|
||||
integrated_world_state_gate_evaluated: true,
|
||||
integrated_world_state_gate_passed: true,
|
||||
detector_replacement_authorized: false,
|
||||
load_envelope_evaluated: true,
|
||||
production_rate_repeatability_passed: true,
|
||||
reserve_12_fps_passed: true,
|
||||
limit_15_fps_passed: true,
|
||||
load_envelope_accepted: true,
|
||||
production_accepted: false,
|
||||
},
|
||||
limitations: ["No independent semantic ground truth."],
|
||||
@@ -175,11 +282,71 @@ test("M4.8S result exposes complete graph load without production authority", as
|
||||
assert.equal(result.metrics.integratedWorldState.deliveredWorldStates, 4481);
|
||||
assert.equal(result.metrics.integratedWorldState.worldStateCompletionAgeP95Ms, 74.733648);
|
||||
assert.equal(result.metrics.integratedWorldState.additionalInferencePasses, 0);
|
||||
assert.equal(result.metrics.runtimeHardening.baseline.supersededFrames, 9);
|
||||
assert.equal(result.metrics.runtimeHardening.hardened.supersededFrames, 1);
|
||||
assert.equal(result.metrics.runtimeHardening.startup.prewarmed.worldStateMs, 60.516957);
|
||||
assert.equal(result.metrics.loadEnvelope.scenarios[0].operatingTargetGatePassed, true);
|
||||
assert.equal(result.metrics.loadEnvelope.scenarios[1].deliveredWorldStates, 4488);
|
||||
assert.equal(result.metrics.loadEnvelope.scenarios[2].repetitions.length, 3);
|
||||
assert.equal(result.metrics.loadEnvelope.scenarios[0].hotLoopDecodeP95Ms, 14);
|
||||
assert.equal(result.metrics.loadEnvelope.computeCapacityAtLeastFps, 15);
|
||||
assert.equal(result.decision.loadEnvelopeAccepted, true);
|
||||
assert.equal(result.decision.integratedWorldStateGatePassed, true);
|
||||
assert.equal(result.decision.productionAccepted, false);
|
||||
assert.equal(result.authority.navigationOrSafetyAccepted, false);
|
||||
});
|
||||
|
||||
test("M4.8S keeps the immutable v1 load-envelope result readable", async () => {
|
||||
const payload = resultPayload();
|
||||
payload.metrics.load_envelope = {
|
||||
schema_version: "missioncore.m48s-load-envelope-comparison/v1",
|
||||
production_rate_repeatability_passed: false,
|
||||
reserve_12_fps_passed: true,
|
||||
limit_15_fps_passed: true,
|
||||
load_envelope_accepted: false,
|
||||
compute_capacity_at_least_fps: 15,
|
||||
bottleneck_interpretation: "rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation",
|
||||
scenarios: payload.metrics.load_envelope.scenarios.map((scenario, index) => ({
|
||||
id: scenario.id,
|
||||
load_purpose: scenario.load_purpose,
|
||||
requested_source_rate_hz: scenario.requested_source_rate_hz,
|
||||
source_frames_admitted: scenario.source_frames_admitted,
|
||||
delivered_world_states: index === 0 ? 4483 : scenario.delivered_world_states,
|
||||
superseded_frames: index === 0 ? 6 : scenario.superseded_frames,
|
||||
delivery_ratio: index === 0 ? 4483 / 4489 : scenario.delivery_ratio,
|
||||
effective_world_state_fps: scenario.effective_world_state_fps,
|
||||
world_state_completion_age_p95_ms: scenario.world_state_completion_age_p95_ms,
|
||||
world_state_completion_age_p99_ms: scenario.world_state_completion_age_p99_ms,
|
||||
world_state_completion_age_maximum_ms: scenario.world_state_completion_age_maximum_ms,
|
||||
decode_p95_ms: scenario.hot_loop_decode_p95_ms,
|
||||
decode_maximum_ms: 3200,
|
||||
detector_p95_ms: scenario.detector_p95_ms,
|
||||
detector_maximum_ms: scenario.detector_maximum_ms,
|
||||
gpu_utilization_mean_percent: scenario.gpu_utilization_mean_percent,
|
||||
gpu_utilization_maximum_percent: scenario.gpu_utilization_maximum_percent,
|
||||
gpu_memory_maximum_mib: scenario.gpu_memory_maximum_mib,
|
||||
process_peak_rss_mib: scenario.process_peak_rss_mib,
|
||||
queue_high_watermarks: scenario.queue_high_watermarks,
|
||||
queue_capacity: 2,
|
||||
additional_inference_passes: 0,
|
||||
integrity_gate_passed: true,
|
||||
operating_target_gate_passed: index !== 0,
|
||||
thresholds: scenario.thresholds,
|
||||
frame_evidence_sha256: "8".repeat(64),
|
||||
})),
|
||||
};
|
||||
payload.decision.production_rate_repeatability_passed = false;
|
||||
payload.decision.load_envelope_accepted = false;
|
||||
|
||||
const result = await fetchM48SFixedClassDetectorResult(resultId, {
|
||||
fetcher: async () => response(payload),
|
||||
});
|
||||
|
||||
assert.equal(result.metrics.loadEnvelope.repeatCountPerRate, 1);
|
||||
assert.equal(result.metrics.loadEnvelope.loadEnvelopeAccepted, false);
|
||||
assert.equal(result.metrics.loadEnvelope.scenarios[0].operatingTargetGatePassed, false);
|
||||
});
|
||||
|
||||
test("M4.8S frame binds exact camera endpoint and risk-only boxes", async () => {
|
||||
const frame = await fetchM48SFixedClassDetectorFrame(resultId, "000253", {
|
||||
fetcher: async () => response({
|
||||
|
||||
@@ -249,7 +249,7 @@
|
||||
},
|
||||
{
|
||||
"catalog_id": "m48s-fixed-class-detector",
|
||||
"evidence_id": "m48s-fixed-class-detector-lab-d1bac05a9e43d407b0f931105cc0e84183ef9ff37666911f03c41486beeb7ef9",
|
||||
"evidence_id": "m48s-fixed-class-detector-lab-4a901d811f53734540337f8d1f2c01666539aa0a9b7786f26d4f1f2e08cc858a",
|
||||
"signal": "progress",
|
||||
"lifecycle": "current",
|
||||
"visual_evidence": "available"
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
{
|
||||
"schema_version": "missioncore.m48s-load-envelope-profile/v2",
|
||||
"profile_id": "m48s-rf-detr-reference-graph-load-envelope-prefetch/v1",
|
||||
"decision_question": "Does bounded recorded-image prefetch remove decode-startup interference and make the 10, 12 and 15 FPS complete-graph results repeatable on Worker 006?",
|
||||
"hypothesis": "A 64-frame bounded decoder prefetch completed before source admission isolates cold video open and initial decode from the realtime hot loop. Three complete repetitions at each rate will retain exact terminal accounting, satisfy the predeclared targets and attribute any residual tail to hot-loop decode, source pacing or graph completion.",
|
||||
"source": {
|
||||
"source_id": "RAVNOVES00",
|
||||
"source_profile_id": "m4-ravnoves00-recorded-realtime/v1",
|
||||
"frame_count": 4489,
|
||||
"graph_id": "reference-perception-graph/v2",
|
||||
"detector_provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0",
|
||||
"sensor_timestamps_preserved": true
|
||||
},
|
||||
"controlled_change_from_v1": {
|
||||
"change": "bounded 64-frame recorded-image decode prefetch before source admission plus source pacing attribution",
|
||||
"unchanged": [
|
||||
"source frames and source timestamps",
|
||||
"RF-DETR TensorRT engine and threshold",
|
||||
"geometry, temporal, motion, rolling-map and threat providers",
|
||||
"latest-wins queue capacities",
|
||||
"single inference pass per admitted detector frame",
|
||||
"detector prewarm and cyclic-GC hot-loop policy"
|
||||
]
|
||||
},
|
||||
"series_control": {
|
||||
"repeat_count_per_rate": 3,
|
||||
"execution_order": [
|
||||
"production-10fps-a1",
|
||||
"reserve-12fps-a1",
|
||||
"limit-15fps-a1",
|
||||
"production-10fps-a2",
|
||||
"reserve-12fps-a2",
|
||||
"limit-15fps-a2",
|
||||
"production-10fps-a3",
|
||||
"reserve-12fps-a3",
|
||||
"limit-15fps-a3"
|
||||
],
|
||||
"worker": "DESKTOP-OPJ8J04 / Worker 006",
|
||||
"execution": "strictly sequential; one graph container and one candidate Triton container per run"
|
||||
},
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "production-10fps",
|
||||
"run_ids": [
|
||||
"m48s-load-prefetch-v1-production-10fps-a1",
|
||||
"m48s-load-prefetch-v1-production-10fps-a2",
|
||||
"m48s-load-prefetch-v1-production-10fps-a3"
|
||||
],
|
||||
"load_purpose": "production-rate",
|
||||
"source_rate_hz": 10.0,
|
||||
"thresholds": {
|
||||
"minimum_delivery_ratio": 0.999,
|
||||
"minimum_effective_world_state_fps": 9.5,
|
||||
"maximum_world_state_completion_p95_ms": 125.0
|
||||
},
|
||||
"required_for_reserve_decision": true
|
||||
},
|
||||
{
|
||||
"id": "reserve-12fps",
|
||||
"run_ids": [
|
||||
"m48s-load-prefetch-v1-reserve-12fps-a1",
|
||||
"m48s-load-prefetch-v1-reserve-12fps-a2",
|
||||
"m48s-load-prefetch-v1-reserve-12fps-a3"
|
||||
],
|
||||
"load_purpose": "reserve-gate",
|
||||
"source_rate_hz": 12.0,
|
||||
"thresholds": {
|
||||
"minimum_delivery_ratio": 0.995,
|
||||
"minimum_effective_world_state_fps": 11.4,
|
||||
"maximum_world_state_completion_p95_ms": 150.0
|
||||
},
|
||||
"required_for_reserve_decision": true
|
||||
},
|
||||
{
|
||||
"id": "limit-15fps",
|
||||
"run_ids": [
|
||||
"m48s-load-prefetch-v1-limit-15fps-a1",
|
||||
"m48s-load-prefetch-v1-limit-15fps-a2",
|
||||
"m48s-load-prefetch-v1-limit-15fps-a3"
|
||||
],
|
||||
"load_purpose": "limit-discovery",
|
||||
"source_rate_hz": 15.0,
|
||||
"thresholds": {
|
||||
"minimum_delivery_ratio": 0.95,
|
||||
"minimum_effective_world_state_fps": 14.25,
|
||||
"maximum_world_state_completion_p95_ms": 175.0
|
||||
},
|
||||
"required_for_reserve_decision": false
|
||||
}
|
||||
],
|
||||
"common_integrity_gates": [
|
||||
"closed terminal accounting",
|
||||
"zero failed, stale, rejected or unavailable frames",
|
||||
"all queue high-watermarks at or below capacity two",
|
||||
"complete single-pass pipeline timing",
|
||||
"detector prewarm before source admission",
|
||||
"source prefetch ready before source admission",
|
||||
"source pacing attribution for every admitted frame",
|
||||
"cyclic GC disabled only during the hot loop and restored after",
|
||||
"all command, actuation, navigation and safety authority remains false"
|
||||
],
|
||||
"exit_decision": {
|
||||
"repeatability_accepted_when": "All three repetitions at 10 and 12 FPS pass every integrity and operating-target gate; 15 FPS remains limit evidence and must pass integrity.",
|
||||
"optimization_scope_when_rejected": "Optimize only the phase identified by preadmission/hot-loop decode and source-pacing attribution; do not add another detector or inference pass."
|
||||
},
|
||||
"authority": {
|
||||
"candidate_accepted": false,
|
||||
"commands_enabled": false,
|
||||
"actuation_allowed": false,
|
||||
"navigation_or_safety_accepted": false,
|
||||
"production_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"schema_version": "missioncore.m48s-load-envelope-profile/v1",
|
||||
"profile_id": "m48s-rf-detr-reference-graph-load-envelope/v1",
|
||||
"decision_question": "Can the unchanged single-pass RF-DETR reference graph sustain the recorded production rate and 20 percent reserve on Worker 006, and where does its bounded latest-wins limit begin?",
|
||||
"hypothesis": "The hardened single-pass graph sustains 10 FPS and 12 FPS without unbounded queues, failed frames, a second inference pass, or more than the predeclared delivery loss; 15 FPS is measured only to locate the limit.",
|
||||
"source": {
|
||||
"source_id": "RAVNOVES00",
|
||||
"source_profile_id": "m4-ravnoves00-recorded-realtime/v1",
|
||||
"frame_count": 4489,
|
||||
"graph_id": "reference-perception-graph/v2",
|
||||
"detector_provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0",
|
||||
"sensor_timestamps_preserved": true
|
||||
},
|
||||
"controlled_change": {
|
||||
"only_variable": "wall_clock_source_rate_hz",
|
||||
"unchanged": [
|
||||
"source frames and source timestamps",
|
||||
"RF-DETR TensorRT engine and threshold",
|
||||
"geometry, temporal, motion, rolling-map and threat providers",
|
||||
"latest-wins queue capacities",
|
||||
"single inference pass per admitted detector frame",
|
||||
"detector prewarm and cyclic-GC hot-loop policy"
|
||||
]
|
||||
},
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "production-10fps",
|
||||
"run_id": "m48s-load-envelope-v1-production-10fps-a1",
|
||||
"load_purpose": "production-rate",
|
||||
"source_rate_hz": 10.0,
|
||||
"thresholds": {
|
||||
"minimum_delivery_ratio": 0.999,
|
||||
"minimum_effective_world_state_fps": 9.5,
|
||||
"maximum_world_state_completion_p95_ms": 125.0
|
||||
},
|
||||
"required_for_reserve_decision": true
|
||||
},
|
||||
{
|
||||
"id": "reserve-12fps",
|
||||
"run_id": "m48s-load-envelope-v1-reserve-12fps-a1",
|
||||
"load_purpose": "reserve-gate",
|
||||
"source_rate_hz": 12.0,
|
||||
"thresholds": {
|
||||
"minimum_delivery_ratio": 0.995,
|
||||
"minimum_effective_world_state_fps": 11.4,
|
||||
"maximum_world_state_completion_p95_ms": 150.0
|
||||
},
|
||||
"required_for_reserve_decision": true
|
||||
},
|
||||
{
|
||||
"id": "limit-15fps",
|
||||
"run_id": "m48s-load-envelope-v1-limit-15fps-a1",
|
||||
"load_purpose": "limit-discovery",
|
||||
"source_rate_hz": 15.0,
|
||||
"thresholds": {
|
||||
"minimum_delivery_ratio": 0.95,
|
||||
"minimum_effective_world_state_fps": 14.25,
|
||||
"maximum_world_state_completion_p95_ms": 175.0
|
||||
},
|
||||
"required_for_reserve_decision": false
|
||||
}
|
||||
],
|
||||
"common_integrity_gates": [
|
||||
"closed terminal accounting",
|
||||
"zero failed, stale, rejected or unavailable frames",
|
||||
"all queue high-watermarks at or below capacity two",
|
||||
"complete single-pass pipeline timing",
|
||||
"detector prewarm before source admission",
|
||||
"cyclic GC disabled only during the hot loop and restored after",
|
||||
"all command, actuation, navigation and safety authority remains false"
|
||||
],
|
||||
"visual_evidence": {
|
||||
"binding": "reuse exact M4.8S full camera plus LiDAR plus 3D/PLAN timeline",
|
||||
"reason": "Only wall-clock pacing changes; source frames, sensor timestamps and graph semantics remain immutable.",
|
||||
"independent_ground_truth": false
|
||||
},
|
||||
"exit_decision": {
|
||||
"reserve_accepted_when": "Both production-10fps and reserve-12fps pass every integrity and operating target gate.",
|
||||
"optimization_scope_when_rejected": "Optimize only the measured bottleneck stage; do not add another detector or inference pass.",
|
||||
"limit_discovery_interpretation": "The 15 FPS result is valid evidence when integrity gates pass even if its operating target gate fails."
|
||||
},
|
||||
"authority": {
|
||||
"candidate_accepted": false,
|
||||
"commands_enabled": false,
|
||||
"actuation_allowed": false,
|
||||
"navigation_or_safety_accepted": false,
|
||||
"production_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
# Mission Core backend lifecycle audit — 2026-08-25
|
||||
|
||||
## Outcome
|
||||
|
||||
The observed outage was operator-tool induced, not an unexplained Python crash,
|
||||
GPU out-of-memory event or host memory leak. A Codex session repeatedly used
|
||||
forced LaunchAgent restarts while the LAB camera/timeline endpoints still had
|
||||
active work. The final forced termination left the old Uvicorn child draining
|
||||
while a replacement tried to acquire the same singleton service lease.
|
||||
|
||||
The failure class is a **process-lifecycle termination leak**: the old process
|
||||
remained alive beyond the restart command's assumption. It is not evidence of
|
||||
unbounded heap growth.
|
||||
|
||||
## Evidence and causal chain
|
||||
|
||||
The private Codex rollout journal
|
||||
`~/.codex/sessions/2026/08/25/rollout-2026-08-25T01-08-27-01a035d1-587d-7112-9506-7cc801c2863c.jsonl`
|
||||
for the active 2026-08-25 task records
|
||||
`launchctl kickstart -k` against `com.nodedc.mission-core.local` at 14:40,
|
||||
15:38, 15:44, 16:22, 16:32, 17:14, 17:29, 17:31 and 17:42 Moscow time. At
|
||||
17:15 it additionally records `SIGTERM`, a ten-second wait, then `SIGKILL`
|
||||
against the exact old process before another kickstart.
|
||||
|
||||
The user-visible 17:09 Moscow-time outage occurred after the 16:32 forced
|
||||
restart and before the later 17:15 TERM/KILL recovery attempt. This ordering
|
||||
rules out the later kill as the start of that outage while still attributes the
|
||||
failure window to the same repeated forced-restart sequence.
|
||||
|
||||
The LaunchAgent evidence showed 139 historical runs and last exit code 143
|
||||
(`SIGTERM`), with no jetsam/OOM record. The application log showed Uvicorn
|
||||
entering graceful shutdown and waiting for connections/background tasks while
|
||||
M4.8S LAB camera/timeline requests were open. Replacement processes reported
|
||||
that Mission Core was already starting or stopping because the old child still
|
||||
held `.runtime/mission-core/.serve.lock`.
|
||||
|
||||
The complete causal chain was:
|
||||
|
||||
```text
|
||||
Codex forced launchctl restart
|
||||
-> SIGTERM reached the uv/Uvicorn generation
|
||||
-> Uvicorn waited without a configured graceful-shutdown deadline
|
||||
-> old child retained the singleton flock during active LAB work
|
||||
-> launchd observed its wrapper transition and attempted a replacement
|
||||
-> replacement failed closed on the singleton lease
|
||||
-> port 8000 remained unavailable until the old generation was killed
|
||||
```
|
||||
|
||||
The Codex agent caused the outage. The backend did not spontaneously fall over.
|
||||
|
||||
## Code and runtime audit
|
||||
|
||||
The audit covered Python service startup/shutdown, ASGI lifespan cleanup,
|
||||
thread joins, subprocess calls, the LaunchAgent declaration and operator
|
||||
restart paths under `src/k1link` and `scripts`.
|
||||
|
||||
| Finding | Severity | State | Resolution |
|
||||
| --- | --- | --- | --- |
|
||||
| MC-LIFE-001: Uvicorn graceful drain had no deadline | P0 | fixed | `timeout_graceful_shutdown=10` |
|
||||
| MC-LIFE-002: PID-only launchd supervision could not detect a live unhealthy service | P0 | fixed | exact-health self-watchdog, three-failure gate, TERM then KILL |
|
||||
| MC-LIFE-003: forced restart did not prove old label/process release | P0 | fixed | SHA-bound plan/apply, full `bootout` disappearance wait, health acceptance and rollback |
|
||||
| MC-LIFE-004: dependency resolution could mutate or delay a recovery launch | P1 | fixed | canonical launcher uses `uv run --no-sync` |
|
||||
| MC-LIFE-004A: launcher parent exit could leave its Python child generation | P1 | fixed | `AbandonProcessGroup=false` makes launchd own the complete group |
|
||||
| MC-LIFE-005: ASGI/plugin close functions can individually block | P1 | bounded externally | Uvicorn 10 s, watchdog 12 s escalation and launchd 20 s deadline bound the whole generation |
|
||||
| MC-LIFE-006: self-health state previously had no separate durable evidence | P1 | fixed | private rotating JSONL watchdog journal |
|
||||
| MC-LIFE-007: three unbounded joins exist in the offline E33 qualification runner | P2 | isolated | daemon-only offline worker path; not imported or executed by the backend lifecycle |
|
||||
| MC-LIFE-008: artifact-build/guardrail scripts contain subprocess calls without local deadlines | P3 | isolated | developer/CI paths only; not service-reachable and cannot hold port 8000 |
|
||||
|
||||
All subprocess calls reachable through the backend probes and compute-network
|
||||
control paths already carry explicit deadlines. Service-owned joins found in
|
||||
the active backend, camera, viewer, preparation, LiDAR-shadow, simulation and
|
||||
protocol lifecycles are bounded. The remaining no-timeout calls identified by
|
||||
the syntax scan are offline artifact/qualification tooling, not request or
|
||||
lifespan paths.
|
||||
|
||||
The direct `.venv/bin/k1link` LaunchAgent entrypoint was also tested and
|
||||
rejected by macOS with `EPERM` while reading `.venv/pyvenv.cfg` below the
|
||||
`Downloads` privacy boundary. The apply tool restored the previous plist and
|
||||
health. The accepted declaration therefore retains the already-authorized
|
||||
Homebrew `uv` boundary and disables syncing; it does not weaken macOS privacy
|
||||
controls.
|
||||
|
||||
## Recovery and resume semantics
|
||||
|
||||
`KeepAlive` now restores a dead process. The self-watchdog converts a
|
||||
live-but-unhealthy event-loop/application generation into a bounded process
|
||||
exit so `KeepAlive` can act. Startup reconstructs read-only plugin runtimes,
|
||||
catalogs and background reconciliation from durable artifacts.
|
||||
|
||||
Auto-resume is intentionally selective:
|
||||
|
||||
- idempotent read/catalog/preparation reconciliation restarts automatically;
|
||||
- browser WebSockets reconnect to the new generation;
|
||||
- interrupted physical acquisition is reconciled or marked interrupted from
|
||||
durable ledgers;
|
||||
- physical commands, acquisition continuation, navigation and actuation are
|
||||
never silently resumed.
|
||||
|
||||
This distinction prevents availability recovery from becoming an authority
|
||||
escalation.
|
||||
|
||||
## Qualification evidence
|
||||
|
||||
Focused lifecycle/perception tests passed before deployment. The installed
|
||||
LaunchAgent accepted exact health with watchdog enabled and a 20-second exit
|
||||
deadline. A controlled termination of the complete Mission Core process group
|
||||
produced a new LaunchAgent PID and exact health in 26.588 seconds without
|
||||
manual intervention. A separate `SIGKILL` crash injection changed PID/PGID
|
||||
`42895` to PID `42942`, restored exact health in 22.996 seconds and left no old
|
||||
group residue.
|
||||
|
||||
The local qualification does not make the current Mac LaunchAgent an onboard
|
||||
deployment artifact. The eventual onboard init declaration must independently
|
||||
prove boot start, crash restart, health-hang restart, power-loss recovery,
|
||||
bounded shutdown, single-generation fencing and fail-closed physical-state
|
||||
reconciliation.
|
||||
@@ -0,0 +1,110 @@
|
||||
# Mission Core local service recovery
|
||||
|
||||
## Scope
|
||||
|
||||
This runbook owns the single local Mission Core backend at
|
||||
`http://127.0.0.1:8000`. It covers startup, health supervision, bounded
|
||||
shutdown and recovery on the current macOS operator station. It does not grant
|
||||
physical acquisition, actuation, navigation or safety authority.
|
||||
|
||||
The canonical LaunchAgent label is:
|
||||
|
||||
```text
|
||||
com.nodedc.mission-core.local
|
||||
```
|
||||
|
||||
Do not start a second backend on another port. Do not use
|
||||
`launchctl kickstart -k` for this service: it combines termination and restart
|
||||
without proving that the old process group has released the singleton lease.
|
||||
|
||||
## Recovery contract
|
||||
|
||||
The installed LaunchAgent and application form one bounded recovery ladder:
|
||||
|
||||
1. `launchd` starts one `uv run --no-sync k1link serve` process group with
|
||||
`RunAtLoad=true`, `KeepAlive=true`, `AbandonProcessGroup=false` and a
|
||||
five-second throttle.
|
||||
2. Mission Core starts a private self-health thread after acquiring the
|
||||
singleton backend lease.
|
||||
3. After the 45-second cold-start grace, three consecutive failed exact
|
||||
`/api/health` probes request `SIGTERM` for the complete process group.
|
||||
4. Uvicorn stops accepting work and has ten seconds to drain active requests
|
||||
and ASGI lifespan work.
|
||||
5. If the service remains alive, the watchdog escalates to `SIGKILL` after
|
||||
twelve seconds. `launchd` also owns a 20-second exit deadline.
|
||||
6. `launchd` starts a fresh process group. Startup reconstructs plugin
|
||||
runtimes and the recording-preparation reconciler from durable state.
|
||||
|
||||
The watchdog journal is private, bounded and rotated:
|
||||
|
||||
```text
|
||||
.runtime/mission-core/service-watchdog.jsonl
|
||||
.runtime/mission-core/service-watchdog.jsonl.1
|
||||
```
|
||||
|
||||
Physical operations are deliberately not resumed from an assumed state.
|
||||
Interrupted preparation and catalog work is reconciled from durable evidence;
|
||||
physical acquisition, commands and actuation remain fail-closed and require a
|
||||
new confirmed authority transition.
|
||||
|
||||
## Plan and apply
|
||||
|
||||
Always plan from the repository root before changing the installed agent:
|
||||
|
||||
```bash
|
||||
uv run python scripts/manage_mission_core_launch_agent.py plan \
|
||||
--repository-root "$PWD"
|
||||
```
|
||||
|
||||
Copy the exact `current_sha256` and `desired_sha256` from that output into the
|
||||
apply command:
|
||||
|
||||
```bash
|
||||
uv run python scripts/manage_mission_core_launch_agent.py apply \
|
||||
--repository-root "$PWD" \
|
||||
--expected-current-sha256 <current-sha256> \
|
||||
--expected-desired-sha256 <desired-sha256>
|
||||
```
|
||||
|
||||
Apply writes a mode-0600 backup below
|
||||
`.runtime/mission-core/launch-agent-backups`, atomically replaces the plist,
|
||||
waits until `bootout` has fully removed the old label, bootstraps the new
|
||||
declaration and accepts only the exact Mission Core health document. Any
|
||||
failure restores the previous plist and repeats the same health acceptance.
|
||||
|
||||
Read-only status:
|
||||
|
||||
```bash
|
||||
uv run python scripts/manage_mission_core_launch_agent.py status \
|
||||
--repository-root "$PWD"
|
||||
```
|
||||
|
||||
## Acceptance after recovery
|
||||
|
||||
The service is recovered only when all of the following are true:
|
||||
|
||||
- `launchctl print gui/$(id -u)/com.nodedc.mission-core.local` reports
|
||||
`state = running`;
|
||||
- the arguments include `uv run --no-sync k1link serve`;
|
||||
- the environment contains `MISSIONCORE_SERVICE_WATCHDOG => 1`;
|
||||
- the launchd exit timeout is 20 seconds;
|
||||
- `GET http://127.0.0.1:8000/api/health` returns HTTP 200 with
|
||||
`ok=true`, `status=ok` and
|
||||
`service=mission-core-control-plane`;
|
||||
- the watchdog journal contains `watchdog-started` for the current child PID;
|
||||
- there is only one `uv` parent and one Mission Core Python child in their
|
||||
exact process group.
|
||||
|
||||
## 2026-08-25 recovery qualification
|
||||
|
||||
The reviewed declaration SHA-256 was
|
||||
`80fca5ec6bdab21f11a544d8dbee65a35f73a6c34752c6a48e9a1181e5da256a`.
|
||||
A controlled `SIGTERM` of the exact Mission Core process group changed the
|
||||
LaunchAgent PID from `40866` to `40957`; exact health recovered automatically
|
||||
in `26.588` seconds without a manual start. After the explicit
|
||||
`AbandonProcessGroup=false` fence was installed, a controlled `SIGKILL` of PID
|
||||
and PGID `42895` produced a new LaunchAgent PID `42942` and exact health in
|
||||
`22.996` seconds, with no old process-group residue. This is local
|
||||
operator-station evidence only. An onboard Linux deployment must express the
|
||||
same contract in its init system and pass its own power-loss, crash, hang and
|
||||
durable-state reconciliation qualification.
|
||||
@@ -47,14 +47,14 @@ from k1link.perception.m48s_reference_graph_runtime import (
|
||||
from k1link.perception.motion import ClassIndependentMotionEstimator
|
||||
from k1link.perception.object_understanding import AdvisoryResponse
|
||||
from k1link.perception.providers import SourcePacket
|
||||
from k1link.perception.recorded_source import DecodedFrameTiming
|
||||
from k1link.perception.recorded_source import DecodedFrameTiming, SourcePacingTiming
|
||||
from k1link.perception.reference_graph_runtime import ReferenceGraphRuntimePaths
|
||||
from k1link.perception.rolling_map import RollingLocalObstacleMapProvider
|
||||
from k1link.perception.temporal import BoundedSpatialTemporalProvider
|
||||
|
||||
SCHEMA_VERSION: Final = "missioncore.m48s-reference-graph-shadow-load/v3"
|
||||
SCHEMA_VERSION: Final = "missioncore.m48s-reference-graph-shadow-load/v5"
|
||||
FRAME_EVIDENCE_SCHEMA: Final = "missioncore.m48s-reference-graph-frame-evidence/v1"
|
||||
PIPELINE_TIMING_SCHEMA: Final = "missioncore.m48s-frame-pipeline-timing/v0"
|
||||
PIPELINE_TIMING_SCHEMA: Final = "missioncore.m48s-frame-pipeline-timing/v1"
|
||||
GC_POLICY_SCHEMA: Final = "missioncore.cyclic-gc-hot-loop-policy/v0"
|
||||
AUTHORITY: Final = {
|
||||
"ground_truth": False,
|
||||
@@ -63,6 +63,7 @@ AUTHORITY: Final = {
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
LOAD_PURPOSES: Final = ("production-rate", "reserve-gate", "limit-discovery")
|
||||
|
||||
|
||||
class GpuTelemetry:
|
||||
@@ -227,14 +228,23 @@ class FrameTimingStore:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._decode: dict[int, int] = {}
|
||||
self._decode: dict[int, DecodedFrameTiming] = {}
|
||||
self._pacing: dict[int, SourcePacingTiming] = {}
|
||||
self._all_decode: list[DecodedFrameTiming] = []
|
||||
self._all_pacing: list[SourcePacingTiming] = []
|
||||
self._detector: dict[int, DetectorFrameTiming] = {}
|
||||
self._providers: dict[int, dict[str, int]] = defaultdict(dict)
|
||||
self._delivered: list[dict[str, object]] = []
|
||||
|
||||
def observe_decode(self, timing: DecodedFrameTiming) -> None:
|
||||
with self._lock:
|
||||
self._decode[timing.sequence] = timing.duration_ns
|
||||
self._decode[timing.sequence] = timing
|
||||
self._all_decode.append(timing)
|
||||
|
||||
def observe_pacing(self, timing: SourcePacingTiming) -> None:
|
||||
with self._lock:
|
||||
self._pacing[timing.sequence] = timing
|
||||
self._all_pacing.append(timing)
|
||||
|
||||
def observe_detector(self, timing: DetectorFrameTiming) -> None:
|
||||
with self._lock:
|
||||
@@ -256,7 +266,8 @@ class FrameTimingStore:
|
||||
) -> dict[str, object]:
|
||||
with self._lock:
|
||||
try:
|
||||
decode_ns = self._decode.pop(sequence)
|
||||
decode = self._decode.pop(sequence)
|
||||
pacing = self._pacing.pop(sequence)
|
||||
detector = self._detector.pop(sequence)
|
||||
providers = self._providers.pop(sequence)
|
||||
except KeyError as exc:
|
||||
@@ -271,13 +282,19 @@ class FrameTimingStore:
|
||||
document = {
|
||||
"schema_version": PIPELINE_TIMING_SCHEMA,
|
||||
"sequence": sequence,
|
||||
"decode_duration_ns": decode_ns,
|
||||
"decode_duration_ns": decode.duration_ns,
|
||||
"decode_phase": decode.phase.value,
|
||||
"source_pacing": {
|
||||
"scheduled_monotonic_ns": pacing.scheduled_monotonic_ns,
|
||||
"emitted_monotonic_ns": pacing.emitted_monotonic_ns,
|
||||
"lateness_ns": pacing.lateness_ns,
|
||||
},
|
||||
"detector": detector.to_dict(),
|
||||
"providers": dict(sorted(providers.items())),
|
||||
"graph_admission_to_delivery_ns": admission_to_delivery_ns,
|
||||
"graph_attributed_provider_ns": attributed_graph_ns,
|
||||
"graph_unattributed_ns": unattributed_ns,
|
||||
"decode_to_delivery_processing_ns": decode_ns + admission_to_delivery_ns,
|
||||
"decode_to_delivery_processing_ns": decode.duration_ns + admission_to_delivery_ns,
|
||||
}
|
||||
with self._lock:
|
||||
self._delivered.append(document)
|
||||
@@ -287,6 +304,14 @@ class FrameTimingStore:
|
||||
with self._lock:
|
||||
return tuple(self._delivered)
|
||||
|
||||
def all_decode(self) -> tuple[DecodedFrameTiming, ...]:
|
||||
with self._lock:
|
||||
return tuple(self._all_decode)
|
||||
|
||||
def all_pacing(self) -> tuple[SourcePacingTiming, ...]:
|
||||
with self._lock:
|
||||
return tuple(self._all_pacing)
|
||||
|
||||
|
||||
class TimedProviderProxy:
|
||||
"""Record one provider's actual call duration without another inference pass."""
|
||||
@@ -359,6 +384,13 @@ def main() -> int:
|
||||
parser.add_argument("--triton-origin", default="http://127.0.0.1:8000")
|
||||
parser.add_argument("--loops", type=int, default=1)
|
||||
parser.add_argument("--maximum-frames", type=int)
|
||||
parser.add_argument("--source-rate-hz", type=float)
|
||||
parser.add_argument("--minimum-delivery-ratio", type=float, default=0.0)
|
||||
parser.add_argument("--minimum-effective-world-state-fps", type=float, default=9.5)
|
||||
parser.add_argument("--maximum-world-state-completion-p95-ms", type=float, default=175.0)
|
||||
parser.add_argument("--load-purpose", choices=LOAD_PURPOSES, default="production-rate")
|
||||
parser.add_argument("--runtime-artifact-sha256", required=True)
|
||||
parser.add_argument("--runner-sha256", required=True)
|
||||
parser.add_argument("--telemetry-interval-seconds", type=float, default=1.0)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--progress", type=Path, required=True)
|
||||
@@ -370,6 +402,21 @@ def main() -> int:
|
||||
raise RuntimeError("maximum frame count must be positive")
|
||||
if arguments.telemetry_interval_seconds <= 0:
|
||||
raise RuntimeError("telemetry interval must be positive")
|
||||
if arguments.source_rate_hz is not None and (
|
||||
not np.isfinite(arguments.source_rate_hz) or arguments.source_rate_hz <= 0
|
||||
):
|
||||
raise RuntimeError("source rate must be positive and finite")
|
||||
if not 0.0 <= arguments.minimum_delivery_ratio <= 1.0:
|
||||
raise RuntimeError("minimum delivery ratio must be between zero and one")
|
||||
if arguments.minimum_effective_world_state_fps <= 0:
|
||||
raise RuntimeError("minimum effective world-state FPS must be positive")
|
||||
if arguments.maximum_world_state_completion_p95_ms <= 0:
|
||||
raise RuntimeError("maximum completion p95 must be positive")
|
||||
for digest in (arguments.runtime_artifact_sha256, arguments.runner_sha256):
|
||||
if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest):
|
||||
raise RuntimeError("runtime and runner SHA-256 values must be lowercase hex")
|
||||
if _sha256(Path(__file__)) != arguments.runner_sha256:
|
||||
raise RuntimeError("runner SHA-256 changed")
|
||||
output = arguments.output.absolute()
|
||||
progress = arguments.progress.absolute()
|
||||
frame_ledger = arguments.frame_ledger.absolute()
|
||||
@@ -397,6 +444,8 @@ def main() -> int:
|
||||
map_output_ages_ms: list[float] = []
|
||||
all_deliveries: list[DeliveredFrame] = []
|
||||
all_pipeline_timings: list[dict[str, object]] = []
|
||||
all_decode_timings: list[DecodedFrameTiming] = []
|
||||
all_pacing_timings: list[SourcePacingTiming] = []
|
||||
with (
|
||||
progress.open("x", encoding="utf-8") as progress_stream,
|
||||
frame_ledger.open("x", encoding="utf-8") as frame_ledger_stream,
|
||||
@@ -424,8 +473,10 @@ def main() -> int:
|
||||
timing_store,
|
||||
),
|
||||
decode_timing_observer=timing_store.observe_decode,
|
||||
source_pacing_observer=timing_store.observe_pacing,
|
||||
detector_timing_observer=timing_store.observe_detector,
|
||||
maximum_frames=arguments.maximum_frames,
|
||||
source_rate_hz=arguments.source_rate_hz,
|
||||
) as runtime:
|
||||
for stage_id, attribute in (
|
||||
("geometry", "geometry"),
|
||||
@@ -445,8 +496,10 @@ def main() -> int:
|
||||
),
|
||||
)
|
||||
detector_warmup = runtime.warm_up_detector()
|
||||
source_prefetch = runtime.prepare_source()
|
||||
gc_policy = CyclicGcHotLoopPolicy()
|
||||
with gc_policy:
|
||||
runtime.mark_source_admission_started()
|
||||
loop_started_ns = time.monotonic_ns()
|
||||
result = runtime.graph.run()
|
||||
loop_completed_ns = time.monotonic_ns()
|
||||
@@ -499,6 +552,7 @@ def main() -> int:
|
||||
setup_seconds=setup_seconds,
|
||||
gc_policy=gc_policy.to_dict(),
|
||||
detector_warmup=detector_warmup,
|
||||
source_prefetch=asdict(source_prefetch),
|
||||
)
|
||||
loop_documents.append(loop_document)
|
||||
completion_ages_ms.extend(value / 1_000_000.0 for value in loop_completion_ages_ns)
|
||||
@@ -507,6 +561,8 @@ def main() -> int:
|
||||
)
|
||||
all_deliveries.extend(result.deliveries)
|
||||
all_pipeline_timings.extend(loop_pipeline_timings)
|
||||
all_decode_timings.extend(timing_store.all_decode())
|
||||
all_pacing_timings.extend(timing_store.all_pacing())
|
||||
frame_ledger_stream.flush()
|
||||
progress_row = {
|
||||
"loop": loop_index + 1,
|
||||
@@ -546,7 +602,9 @@ def main() -> int:
|
||||
identity = _identity_metrics(all_deliveries)
|
||||
semantic = _semantic_metrics(all_deliveries, advisories)
|
||||
world_state_fps = delivered / processing_wall_seconds
|
||||
checks = {
|
||||
delivery_ratio = delivered / admitted if admitted else 0.0
|
||||
completion_distribution = _distribution(completion_ages_ms)
|
||||
integrity_checks = {
|
||||
"loop_count_completed": len(loop_documents) == arguments.loops,
|
||||
"graph_stopped_cleanly": all(
|
||||
loop["state"] == GraphState.STOPPED.value for loop in loop_documents
|
||||
@@ -561,10 +619,6 @@ def main() -> int:
|
||||
TerminalOutcomeType.UNAVAILABLE,
|
||||
)
|
||||
),
|
||||
"minimum_world_state_fps": world_state_fps >= 9.5,
|
||||
"maximum_world_state_completion_p95_ms": (
|
||||
_distribution(completion_ages_ms)["p95"] <= 175.0
|
||||
),
|
||||
"bounded_latest_wins_queues": all(value <= 2 for value in queue_high_watermarks.values()),
|
||||
"temporal_identity_reuse_observed": cast(
|
||||
int,
|
||||
@@ -586,8 +640,27 @@ def main() -> int:
|
||||
and cast(dict[str, object], loop["detector_warmup"])["inference_passes"] == 1
|
||||
for loop in loop_documents
|
||||
),
|
||||
"source_prefetch_completed_before_source_admission": all(
|
||||
cast(dict[str, object], loop["source_prefetch"])["buffered_frames"]
|
||||
== cast(dict[str, object], loop["source_prefetch"])["ready_frames"]
|
||||
for loop in loop_documents
|
||||
),
|
||||
"source_pacing_attribution_complete": len(all_pacing_timings) == admitted,
|
||||
"authority_remains_false": all(value is False for value in AUTHORITY.values()),
|
||||
}
|
||||
operating_target_checks = {
|
||||
"minimum_delivery_ratio": delivery_ratio >= arguments.minimum_delivery_ratio,
|
||||
"minimum_world_state_fps": (
|
||||
world_state_fps >= arguments.minimum_effective_world_state_fps
|
||||
),
|
||||
"maximum_world_state_completion_p95_ms": (
|
||||
completion_distribution["p95"]
|
||||
<= arguments.maximum_world_state_completion_p95_ms
|
||||
),
|
||||
}
|
||||
checks = {**integrity_checks, **operating_target_checks}
|
||||
evidence_integrity_gate_passed = all(integrity_checks.values())
|
||||
operating_target_gate_passed = all(operating_target_checks.values())
|
||||
integrated_runtime_gate_passed = all(checks.values())
|
||||
document = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
@@ -595,20 +668,28 @@ def main() -> int:
|
||||
"source_id": "RAVNOVES00",
|
||||
"loops": arguments.loops,
|
||||
"maximum_frames_per_loop": arguments.maximum_frames,
|
||||
"requested_rate_hz": arguments.source_rate_hz,
|
||||
"pacing_contract": "wall-clock-scaled-source-timestamps-immutable/v1",
|
||||
},
|
||||
"identity": {
|
||||
"worker_id": "worker-006",
|
||||
"graph_id": "reference-perception-graph/v2",
|
||||
"detector_provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0",
|
||||
"inputs": _input_digests(paths, arguments.detector_profile),
|
||||
"runtime_artifact_sha256": arguments.runtime_artifact_sha256,
|
||||
"runner_sha256": arguments.runner_sha256,
|
||||
},
|
||||
"execution": {
|
||||
"run_mode": GraphRunMode.SOURCE_PACED_LATEST_WINS.value,
|
||||
"load_purpose": arguments.load_purpose,
|
||||
"requested_source_rate_hz": arguments.source_rate_hz,
|
||||
"source_timestamps_preserved": True,
|
||||
"wall_seconds": round(wall_seconds, 6),
|
||||
"source_processing_wall_seconds": round(processing_wall_seconds, 6),
|
||||
"admitted_frames": admitted,
|
||||
"delivered_world_states": delivered,
|
||||
"effective_world_state_fps": round(world_state_fps, 6),
|
||||
"delivery_ratio": round(delivery_ratio, 9),
|
||||
"terminal_outcomes": dict(sorted(accounting.items())),
|
||||
"queue_high_watermarks": queue_high_watermarks,
|
||||
"loops": loop_documents,
|
||||
@@ -624,17 +705,32 @@ def main() -> int:
|
||||
},
|
||||
},
|
||||
"metrics": {
|
||||
"world_state_completion_age_ms": _distribution(completion_ages_ms),
|
||||
"world_state_completion_age_ms": completion_distribution,
|
||||
"local_obstacle_map_output_age_ms": _distribution(map_output_ages_ms),
|
||||
"identity_continuity": identity,
|
||||
"semantic_advisory": semantic,
|
||||
"pipeline_timing": _pipeline_timing_metrics(all_pipeline_timings),
|
||||
"source_decode": _source_decode_metrics(all_decode_timings),
|
||||
"source_pacing": _source_pacing_metrics(all_pacing_timings),
|
||||
"python_gc": _gc_telemetry_summary(gc_telemetry.events),
|
||||
"gpu": _telemetry_summary(gpu.samples),
|
||||
"process_peak_rss_before_mib": round(rss_before_kib / 1024.0, 6),
|
||||
"process_peak_rss_after_mib": round(rss_after_kib / 1024.0, 6),
|
||||
},
|
||||
"checks": checks,
|
||||
"integrity_checks": integrity_checks,
|
||||
"operating_target_checks": operating_target_checks,
|
||||
"predeclared_thresholds": {
|
||||
"minimum_delivery_ratio": arguments.minimum_delivery_ratio,
|
||||
"minimum_effective_world_state_fps": (
|
||||
arguments.minimum_effective_world_state_fps
|
||||
),
|
||||
"maximum_world_state_completion_p95_ms": (
|
||||
arguments.maximum_world_state_completion_p95_ms
|
||||
),
|
||||
},
|
||||
"evidence_integrity_gate_passed": evidence_integrity_gate_passed,
|
||||
"operating_target_gate_passed": operating_target_gate_passed,
|
||||
"integrated_runtime_gate_passed": integrated_runtime_gate_passed,
|
||||
"independent_track_identity_quality_evaluated": False,
|
||||
"independent_risk_policy_quality_evaluated": False,
|
||||
@@ -647,7 +743,10 @@ def main() -> int:
|
||||
output.write_bytes(_canonical_json(document) + b"\n")
|
||||
print(output)
|
||||
print(json.dumps(checks, indent=2, sort_keys=True))
|
||||
return 0 if integrated_runtime_gate_passed else 2
|
||||
discovery_completed = (
|
||||
arguments.load_purpose == "limit-discovery" and evidence_integrity_gate_passed
|
||||
)
|
||||
return 0 if integrated_runtime_gate_passed or discovery_completed else 2
|
||||
|
||||
|
||||
def _record_completion_age(
|
||||
@@ -703,6 +802,7 @@ def _loop_document(
|
||||
setup_seconds: float,
|
||||
gc_policy: dict[str, object],
|
||||
detector_warmup: DetectorWarmupSnapshot,
|
||||
source_prefetch: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
outcomes = Counter(item.outcome.value for item in result.terminal_outcomes)
|
||||
outcome_stages = Counter(
|
||||
@@ -715,6 +815,7 @@ def _loop_document(
|
||||
"setup_seconds": round(setup_seconds, 6),
|
||||
"cyclic_gc_hot_loop": gc_policy,
|
||||
"detector_warmup": asdict(detector_warmup),
|
||||
"source_prefetch": source_prefetch,
|
||||
"admitted_count": result.admitted_count,
|
||||
"delivered_count": len(result.deliveries),
|
||||
"effective_world_state_fps": round(len(result.deliveries) / wall_seconds, 6),
|
||||
@@ -840,6 +941,8 @@ def _pipeline_timing_metrics(
|
||||
"decode_to_delivery_processing_ns",
|
||||
)
|
||||
top_level_values: dict[str, list[float]] = {key: [] for key in top_level_fields}
|
||||
decode_by_phase: dict[str, list[float]] = defaultdict(list)
|
||||
delivered_pacing_lateness_ms: list[float] = []
|
||||
for document in documents:
|
||||
detector = cast(Mapping[str, int], document["detector"])
|
||||
providers = cast(Mapping[str, int], document["providers"])
|
||||
@@ -849,6 +952,11 @@ def _pipeline_timing_metrics(
|
||||
provider_values[key].append(providers[key] / 1_000_000.0)
|
||||
for key in top_level_fields:
|
||||
top_level_values[key].append(cast(int, document[key]) / 1_000_000.0)
|
||||
decode_by_phase[cast(str, document["decode_phase"])].append(
|
||||
cast(int, document["decode_duration_ns"]) / 1_000_000.0
|
||||
)
|
||||
source_pacing = cast(Mapping[str, int], document["source_pacing"])
|
||||
delivered_pacing_lateness_ms.append(source_pacing["lateness_ns"] / 1_000_000.0)
|
||||
maximum = max(
|
||||
documents,
|
||||
key=lambda document: cast(int, document["graph_admission_to_delivery_ns"]),
|
||||
@@ -857,6 +965,12 @@ def _pipeline_timing_metrics(
|
||||
return {
|
||||
"sample_count": len(documents),
|
||||
"decode_duration_ms": _distribution(top_level_values["decode_duration_ns"]),
|
||||
"decode_duration_by_phase_ms": {
|
||||
phase: _distribution(values) for phase, values in sorted(decode_by_phase.items())
|
||||
},
|
||||
"delivered_source_pacing_lateness_ms": _distribution(
|
||||
delivered_pacing_lateness_ms
|
||||
),
|
||||
"detector_ms": {
|
||||
key.removesuffix("_duration_ns"): _distribution(values)
|
||||
for key, values in detector_values.items()
|
||||
@@ -877,6 +991,50 @@ def _pipeline_timing_metrics(
|
||||
}
|
||||
|
||||
|
||||
def _source_decode_metrics(samples: list[DecodedFrameTiming]) -> dict[str, object]:
|
||||
by_phase: dict[str, list[float]] = defaultdict(list)
|
||||
for sample in samples:
|
||||
by_phase[sample.phase.value].append(sample.duration_ns / 1_000_000.0)
|
||||
return {
|
||||
"sample_count": len(samples),
|
||||
"phase_counts": {
|
||||
phase: len(values) for phase, values in sorted(by_phase.items())
|
||||
},
|
||||
"duration_by_phase_ms": {
|
||||
phase: _distribution(values) for phase, values in sorted(by_phase.items())
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _source_pacing_metrics(samples: list[SourcePacingTiming]) -> dict[str, object]:
|
||||
ordered = sorted(samples, key=lambda sample: sample.sequence)
|
||||
lateness_ms = [sample.lateness_ns / 1_000_000.0 for sample in ordered]
|
||||
scheduled_intervals_ms = [
|
||||
(current.scheduled_monotonic_ns - previous.scheduled_monotonic_ns) / 1_000_000.0
|
||||
for previous, current in zip(ordered, ordered[1:], strict=False)
|
||||
]
|
||||
emitted_intervals_ms = [
|
||||
(current.emitted_monotonic_ns - previous.emitted_monotonic_ns) / 1_000_000.0
|
||||
for previous, current in zip(ordered, ordered[1:], strict=False)
|
||||
]
|
||||
catch_up_emissions = sum(
|
||||
emitted < scheduled * 0.5
|
||||
for scheduled, emitted in zip(
|
||||
scheduled_intervals_ms,
|
||||
emitted_intervals_ms,
|
||||
strict=True,
|
||||
)
|
||||
if scheduled > 0
|
||||
)
|
||||
return {
|
||||
"sample_count": len(ordered),
|
||||
"lateness_ms": _distribution(lateness_ms),
|
||||
"scheduled_interval_ms": _distribution(scheduled_intervals_ms),
|
||||
"emitted_interval_ms": _distribution(emitted_intervals_ms),
|
||||
"catch_up_emission_count": catch_up_emissions,
|
||||
}
|
||||
|
||||
|
||||
def _telemetry_summary(samples: list[dict[str, float]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"sample_count": len(samples)}
|
||||
for key in (
|
||||
|
||||
@@ -12,6 +12,16 @@ param(
|
||||
[int]$Loops = 1,
|
||||
[ValidateRange(1, 4489)]
|
||||
[int]$MaximumFrames = 120,
|
||||
[ValidateRange(1.0, 120.0)]
|
||||
[double]$SourceRateHz = 10.0,
|
||||
[ValidateRange(0.0, 1.0)]
|
||||
[double]$MinimumDeliveryRatio = 0.999,
|
||||
[ValidateRange(0.1, 120.0)]
|
||||
[double]$MinimumEffectiveWorldStateFps = 9.5,
|
||||
[ValidateRange(1.0, 10000.0)]
|
||||
[double]$MaximumWorldStateCompletionP95Ms = 125.0,
|
||||
[ValidateSet("production-rate", "reserve-gate", "limit-discovery")]
|
||||
[string]$LoadPurpose = "production-rate",
|
||||
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m48s-reference-graph-shadow"
|
||||
)
|
||||
|
||||
@@ -103,6 +113,7 @@ $runner = Get-Item -LiteralPath (
|
||||
if ($runner.PSIsContainer -or ($runner.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw "M48S graph runner must be a regular file"
|
||||
}
|
||||
$runnerSha256 = Get-Sha256 $runner.FullName
|
||||
|
||||
$experimentRoot = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z"
|
||||
@@ -253,6 +264,13 @@ try {
|
||||
"--valid-fov-mask", "/source/mask.png",
|
||||
"--triton-origin", "http://127.0.0.1:8000",
|
||||
"--loops", ([string]$Loops),
|
||||
"--source-rate-hz", ([string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $SourceRateHz)),
|
||||
"--minimum-delivery-ratio", ([string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $MinimumDeliveryRatio)),
|
||||
"--minimum-effective-world-state-fps", ([string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $MinimumEffectiveWorldStateFps)),
|
||||
"--maximum-world-state-completion-p95-ms", ([string]::Format([Globalization.CultureInfo]::InvariantCulture, "{0:R}", $MaximumWorldStateCompletionP95Ms)),
|
||||
"--load-purpose", $LoadPurpose,
|
||||
"--runtime-artifact-sha256", $ExpectedWheelSha256,
|
||||
"--runner-sha256", $runnerSha256,
|
||||
"--output", "/output/result.json",
|
||||
"--progress", "/output/progress.jsonl",
|
||||
"--frame-ledger", "/output/frames.jsonl"
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plan/apply the user-owned Mission Core LaunchAgent with rollback acceptance."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from contextlib import suppress
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.local_service_launchd import (
|
||||
MISSION_CORE_LAUNCH_AGENT_LABEL,
|
||||
MissionCoreLaunchAgentError,
|
||||
plan_mission_core_launch_agent,
|
||||
)
|
||||
|
||||
_LAUNCHCTL_TRANSITION_TIMEOUT_SECONDS = 30.0
|
||||
_LAUNCHCTL_TRANSITION_POLL_SECONDS = 0.1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=("plan", "apply", "status"))
|
||||
parser.add_argument("--repository-root", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--agent-path",
|
||||
type=Path,
|
||||
default=Path.home()
|
||||
/ "Library/LaunchAgents/com.nodedc.mission-core.local.plist",
|
||||
)
|
||||
parser.add_argument("--expected-current-sha256")
|
||||
parser.add_argument("--expected-desired-sha256")
|
||||
arguments = parser.parse_args()
|
||||
if arguments.action == "status":
|
||||
print(json.dumps(_status(), sort_keys=True))
|
||||
return 0
|
||||
plan = plan_mission_core_launch_agent(
|
||||
repository_root=arguments.repository_root,
|
||||
agent_path=arguments.agent_path,
|
||||
)
|
||||
if arguments.action == "plan":
|
||||
print(json.dumps(plan.to_dict(), indent=2, sort_keys=True))
|
||||
return 0
|
||||
if (
|
||||
arguments.expected_current_sha256 != plan.current_sha256
|
||||
or arguments.expected_desired_sha256 != plan.desired_sha256
|
||||
):
|
||||
raise MissionCoreLaunchAgentError("launch agent plan changed before apply")
|
||||
previous = plan.agent_path.read_bytes()
|
||||
backup_root = (
|
||||
arguments.repository_root.expanduser().resolve()
|
||||
/ ".runtime/mission-core/launch-agent-backups"
|
||||
)
|
||||
backup_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
backup = backup_root / f"{plan.current_sha256}.plist"
|
||||
if backup.exists() and hashlib.sha256(backup.read_bytes()).hexdigest() != plan.current_sha256:
|
||||
raise MissionCoreLaunchAgentError("launch agent backup identity collision")
|
||||
if not backup.exists():
|
||||
_write_atomic(backup, previous)
|
||||
_write_atomic(plan.agent_path, plan.desired_payload)
|
||||
try:
|
||||
_reload_launch_agent(plan.agent_path)
|
||||
accepted = _wait_for_health(60.0)
|
||||
if not accepted:
|
||||
raise MissionCoreLaunchAgentError("reloaded Mission Core did not become healthy")
|
||||
except BaseException:
|
||||
_write_atomic(plan.agent_path, previous)
|
||||
_reload_launch_agent(plan.agent_path)
|
||||
if not _wait_for_health(60.0):
|
||||
raise MissionCoreLaunchAgentError(
|
||||
"launch agent apply failed and rollback did not recover health"
|
||||
) from None
|
||||
raise
|
||||
result = {
|
||||
**plan.to_dict(),
|
||||
"applied": True,
|
||||
"backup_path": str(backup),
|
||||
"health_accepted": True,
|
||||
"status": _status(),
|
||||
}
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def _write_atomic(path: Path, payload: bytes) -> None:
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||
try:
|
||||
os.fchmod(descriptor, 0o600)
|
||||
with os.fdopen(descriptor, "wb", closefd=True) as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
except BaseException:
|
||||
with suppress(OSError):
|
||||
os.close(descriptor)
|
||||
with suppress(FileNotFoundError):
|
||||
os.unlink(temporary)
|
||||
raise
|
||||
|
||||
|
||||
def _reload_launch_agent(path: Path) -> None:
|
||||
"""Reload only after launchd proves the previous job is fully absent.
|
||||
|
||||
``launchctl bootout`` can return while the job is still represented by an
|
||||
``xpcproxy`` transition. An immediate bootstrap is then rejected even
|
||||
though the plist is valid. Waiting on the exact label avoids that race and
|
||||
keeps apply/rollback symmetric.
|
||||
"""
|
||||
|
||||
domain = f"gui/{os.getuid()}"
|
||||
target = f"{domain}/{MISSION_CORE_LAUNCH_AGENT_LABEL}"
|
||||
bootout = subprocess.run(
|
||||
["launchctl", "bootout", f"{domain}/{MISSION_CORE_LAUNCH_AGENT_LABEL}"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30.0,
|
||||
)
|
||||
if bootout.returncode != 0 and _launch_agent_loaded(target):
|
||||
raise MissionCoreLaunchAgentError(
|
||||
f"launchctl bootout failed with exit code {bootout.returncode}"
|
||||
)
|
||||
_wait_until_launch_agent_unloaded(target)
|
||||
completed = subprocess.run(
|
||||
["launchctl", "bootstrap", domain, str(path)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30.0,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise MissionCoreLaunchAgentError(
|
||||
"launchctl bootstrap rejected the Mission Core agent "
|
||||
f"with exit code {completed.returncode}"
|
||||
)
|
||||
|
||||
|
||||
def _launch_agent_loaded(target: str) -> bool:
|
||||
completed = subprocess.run(
|
||||
["launchctl", "print", target],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5.0,
|
||||
)
|
||||
return completed.returncode == 0
|
||||
|
||||
|
||||
def _wait_until_launch_agent_unloaded(target: str) -> None:
|
||||
deadline = time.monotonic() + _LAUNCHCTL_TRANSITION_TIMEOUT_SECONDS
|
||||
while time.monotonic() < deadline:
|
||||
if not _launch_agent_loaded(target):
|
||||
return
|
||||
time.sleep(_LAUNCHCTL_TRANSITION_POLL_SECONDS)
|
||||
raise MissionCoreLaunchAgentError(
|
||||
"Mission Core launch agent did not finish bootout before the deadline"
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_health(timeout_seconds: float) -> bool:
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
while time.monotonic() < deadline:
|
||||
if _healthy():
|
||||
return True
|
||||
time.sleep(0.25)
|
||||
return False
|
||||
|
||||
|
||||
def _healthy() -> bool:
|
||||
connection = http.client.HTTPConnection("127.0.0.1", 8000, timeout=1.0)
|
||||
try:
|
||||
connection.request("GET", "/api/health", headers={"Connection": "close"})
|
||||
response = connection.getresponse()
|
||||
payload = response.read(64 * 1024 + 1)
|
||||
except (OSError, TimeoutError, http.client.HTTPException):
|
||||
return False
|
||||
finally:
|
||||
connection.close()
|
||||
if response.status != 200 or len(payload) > 64 * 1024:
|
||||
return False
|
||||
try:
|
||||
document = json.loads(payload)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return False
|
||||
return bool(
|
||||
isinstance(document, dict)
|
||||
and document.get("ok") is True
|
||||
and document.get("status") == "ok"
|
||||
and document.get("service") == "mission-core-control-plane"
|
||||
)
|
||||
|
||||
|
||||
def _status() -> dict[str, object]:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"launchctl",
|
||||
"print",
|
||||
f"gui/{os.getuid()}/{MISSION_CORE_LAUNCH_AGENT_LABEL}",
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5.0,
|
||||
)
|
||||
return {
|
||||
"label": MISSION_CORE_LAUNCH_AGENT_LABEL,
|
||||
"launchd_loaded": completed.returncode == 0,
|
||||
"health_ok": _healthy(),
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -81,6 +81,7 @@ from k1link.device_plugins.xgrids_k1.protocol.application_authority import (
|
||||
)
|
||||
from k1link.device_plugins.xgrids_k1.usb.snapshot import snapshot as usb_snapshot
|
||||
from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials
|
||||
from k1link.service_watchdog import MissionCoreSelfWatchdog, watchdog_enabled
|
||||
from k1link.sessions import SessionIntegrityError
|
||||
|
||||
app = typer.Typer(
|
||||
@@ -116,6 +117,7 @@ app.add_typer(artifact_app, name="artifact")
|
||||
|
||||
_CANONICAL_MISSION_CORE_PORT = 8000
|
||||
_MISSION_CORE_SERVE_LOCK_FILENAME = ".serve.lock"
|
||||
_MISSION_CORE_GRACEFUL_SHUTDOWN_SECONDS = 10
|
||||
|
||||
|
||||
class _MissionCoreServeLeaseError(RuntimeError):
|
||||
@@ -1166,13 +1168,23 @@ def serve_console(
|
||||
f"NODEDC MISSION CORE: http://127.0.0.1:{_CANONICAL_MISSION_CORE_PORT}"
|
||||
)
|
||||
console.print("The credential endpoint is bound to this Mac only.")
|
||||
uvicorn.run(
|
||||
"k1link.web.app:app",
|
||||
host="127.0.0.1",
|
||||
port=_CANONICAL_MISSION_CORE_PORT,
|
||||
log_level="info",
|
||||
access_log=True,
|
||||
watchdog = (
|
||||
MissionCoreSelfWatchdog(repository_root) if watchdog_enabled() else None
|
||||
)
|
||||
if watchdog is not None:
|
||||
watchdog.start()
|
||||
try:
|
||||
uvicorn.run(
|
||||
"k1link.web.app:app",
|
||||
host="127.0.0.1",
|
||||
port=_CANONICAL_MISSION_CORE_PORT,
|
||||
log_level="info",
|
||||
access_log=True,
|
||||
timeout_graceful_shutdown=_MISSION_CORE_GRACEFUL_SHUTDOWN_SECONDS,
|
||||
)
|
||||
finally:
|
||||
if watchdog is not None:
|
||||
watchdog.stop()
|
||||
|
||||
|
||||
def _print_existing_mission_core(port: int) -> None:
|
||||
|
||||
@@ -21,6 +21,8 @@ LAB_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-lab/v1"
|
||||
CATALOG_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame-catalog/v1"
|
||||
FRAME_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-frame/v1"
|
||||
REPORT_SCHEMA: Final = "missioncore.m48s-fixed-class-detector-report/v1"
|
||||
RUNTIME_HARDENING_SCHEMA: Final = "missioncore.m48s-runtime-hardening-comparison/v1"
|
||||
LOAD_ENVELOPE_SCHEMA: Final = "missioncore.m48s-load-envelope-comparison/v2"
|
||||
METHOD_SCHEMA: Final = "missioncore.laboratory-method/v1"
|
||||
RESULT_PREFIX: Final = "m48s-fixed-class-detector-lab-"
|
||||
TOURNAMENT_ID: Final = (
|
||||
@@ -35,10 +37,154 @@ REFERENCE_GRAPH_ID: Final = (
|
||||
"e8da7a521768daba0ead1a6e4803871ce3a85f91a7d8ee36c5719ac10433e791"
|
||||
)
|
||||
REFERENCE_GRAPH_REPLAY_ID: Final = (
|
||||
"m48s-reference-graph-replay-"
|
||||
"16d69d610c22e6f42071b8378cd75dfa6b95db4ceb800f9c7508fa3673504478"
|
||||
"m48s-reference-graph-replay-16d69d610c22e6f42071b8378cd75dfa6b95db4ceb800f9c7508fa3673504478"
|
||||
)
|
||||
INTEGRATED_STATUS: Final = "complete-reference-graph-shadow-passed-production-not-authorized"
|
||||
RUNTIME_HARDENING_RUNS: Final = {
|
||||
"baseline": {
|
||||
"directory": "m48s-pipeline-timing-c51761b5",
|
||||
"schema_version": "missioncore.m48s-reference-graph-shadow-load/v1",
|
||||
"result_sha256": "277f01a1b6499f08f6fdb65485296c9a6ca214a4f89deaca51d1dc9b00a28528",
|
||||
"frames_sha256": "c870814159f362e3080ad72a44aab31e1210d5b2d2fd5da8390c05dcd30fbcb8",
|
||||
"admitted_count": 4489,
|
||||
"frame_count": 4480,
|
||||
},
|
||||
"hardened": {
|
||||
"directory": "m48s-gc-bounded-477886d",
|
||||
"schema_version": "missioncore.m48s-reference-graph-shadow-load/v2",
|
||||
"result_sha256": "13e4423a89b0c1cc78b0219bf5842c42503951ef1372a6996818b3ed8ad3d3ed",
|
||||
"frames_sha256": "1508da7570fc27aeed010a04ff5eadb6e4a8dedd31e9c617bc021c505a88513d",
|
||||
"admitted_count": 4489,
|
||||
"frame_count": 4488,
|
||||
},
|
||||
"prewarmed": {
|
||||
"directory": "m48s-prewarm-84624ae",
|
||||
"schema_version": "missioncore.m48s-reference-graph-shadow-load/v3",
|
||||
"result_sha256": "5881f86ac01dc4e8886c4daef5bcd9d3510d50d7f4144c0341e0144ebb7015b1",
|
||||
"frames_sha256": "67c041beee642f3ff8591c75c80a9bb56c29ad3d63168b38df19ecc475c99d55",
|
||||
"admitted_count": 1000,
|
||||
"frame_count": 1000,
|
||||
},
|
||||
}
|
||||
LOAD_ENVELOPE_RUNS: Final = {
|
||||
"production-10fps-a1": {
|
||||
"scenario_id": "production-10fps",
|
||||
"repetition": 1,
|
||||
"run_id": "m48s-load-prefetch-v1-production-10fps-a1",
|
||||
"load_purpose": "production-rate",
|
||||
"source_rate_hz": 10.0,
|
||||
"result_sha256": "0cf8b682b3972d434e9e10a6eb68a5abc2c1005d731cd61374de951f13d45282",
|
||||
"frames_sha256": "40d02cb59cb3531b764eface2e5f8b8dbf1d1e4ac83801746a97306ac8363910",
|
||||
"delivered": 4489,
|
||||
"superseded": 0,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"reserve-12fps-a1": {
|
||||
"scenario_id": "reserve-12fps",
|
||||
"repetition": 1,
|
||||
"run_id": "m48s-load-prefetch-v1-reserve-12fps-a1",
|
||||
"load_purpose": "reserve-gate",
|
||||
"source_rate_hz": 12.0,
|
||||
"result_sha256": "70e0a623624ed162387960867d3615139e452ff30a6ba8daebe6965f39d76f58",
|
||||
"frames_sha256": "d54187c1b56e43f06498d2deac26374120a36cfd113c3b8f3e4dc170280353aa",
|
||||
"delivered": 4488,
|
||||
"superseded": 1,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"limit-15fps-a1": {
|
||||
"scenario_id": "limit-15fps",
|
||||
"repetition": 1,
|
||||
"run_id": "m48s-load-prefetch-v1-limit-15fps-a1",
|
||||
"load_purpose": "limit-discovery",
|
||||
"source_rate_hz": 15.0,
|
||||
"result_sha256": "d97ebdcc3bd70dc50284c6482fef614b7ec76180749c32b45c31d5e7e6667133",
|
||||
"frames_sha256": "b1b79205d266585b0d545bb5a02e87c957759774aeacab2f2d5290cb5d735bf7",
|
||||
"delivered": 4488,
|
||||
"superseded": 1,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"production-10fps-a2": {
|
||||
"scenario_id": "production-10fps",
|
||||
"repetition": 2,
|
||||
"run_id": "m48s-load-prefetch-v1-production-10fps-a2",
|
||||
"load_purpose": "production-rate",
|
||||
"source_rate_hz": 10.0,
|
||||
"result_sha256": "c98c052b759cecd390e6e79d9eac223e45c96e532bd974d9961d5904945fbfe8",
|
||||
"frames_sha256": "8574b98aee54ea25e31fc80cbcb7720683bdb36116dc3f2606aed4b0316b8cfa",
|
||||
"delivered": 4489,
|
||||
"superseded": 0,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"reserve-12fps-a2": {
|
||||
"scenario_id": "reserve-12fps",
|
||||
"repetition": 2,
|
||||
"run_id": "m48s-load-prefetch-v1-reserve-12fps-a2",
|
||||
"load_purpose": "reserve-gate",
|
||||
"source_rate_hz": 12.0,
|
||||
"result_sha256": "16cc2cb0fd9a56707910dfb397021613eb57b1a1d6259b52d14ec38a1e348eb6",
|
||||
"frames_sha256": "cabd11a2a363b702e2a345198de12408079532ebaa8a095bcfa458acf763325f",
|
||||
"delivered": 4489,
|
||||
"superseded": 0,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"limit-15fps-a2": {
|
||||
"scenario_id": "limit-15fps",
|
||||
"repetition": 2,
|
||||
"run_id": "m48s-load-prefetch-v1-limit-15fps-a2",
|
||||
"load_purpose": "limit-discovery",
|
||||
"source_rate_hz": 15.0,
|
||||
"result_sha256": "bf23b09b6874f506170448484d1ca2886d8abeb951f9536e810ed24991a0efa6",
|
||||
"frames_sha256": "fe3b37bde43fe92d92e5da4004509ef87aa85b25700cf8d9d5c84817e13b2422",
|
||||
"delivered": 4488,
|
||||
"superseded": 1,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"production-10fps-a3": {
|
||||
"scenario_id": "production-10fps",
|
||||
"repetition": 3,
|
||||
"run_id": "m48s-load-prefetch-v1-production-10fps-a3",
|
||||
"load_purpose": "production-rate",
|
||||
"source_rate_hz": 10.0,
|
||||
"result_sha256": "ebb81d69c16b32aa6f2237a3a5e183def58dcb422b1b82fa2621d340cc1b6fcc",
|
||||
"frames_sha256": "2668b936d274a273dbe440dade55716f686af507aaa4f1a02803396accfe240d",
|
||||
"delivered": 4489,
|
||||
"superseded": 0,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"reserve-12fps-a3": {
|
||||
"scenario_id": "reserve-12fps",
|
||||
"repetition": 3,
|
||||
"run_id": "m48s-load-prefetch-v1-reserve-12fps-a3",
|
||||
"load_purpose": "reserve-gate",
|
||||
"source_rate_hz": 12.0,
|
||||
"result_sha256": "655d3c0b6c7210f12cfb3a9edf84da5851459afa7303c9732b879e454ac05dc2",
|
||||
"frames_sha256": "419e8862f0da0e54a3b82f581230ab72874bf52841f62a4e49e1905ed5a77520",
|
||||
"delivered": 4485,
|
||||
"superseded": 4,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
"limit-15fps-a3": {
|
||||
"scenario_id": "limit-15fps",
|
||||
"repetition": 3,
|
||||
"run_id": "m48s-load-prefetch-v1-limit-15fps-a3",
|
||||
"load_purpose": "limit-discovery",
|
||||
"source_rate_hz": 15.0,
|
||||
"result_sha256": "e844183dcee5982f389c9aa847cd98cec78fb81e0b3920e606063740eb35f421",
|
||||
"frames_sha256": "477cffcecbaf607c8c68979b5425f84963d07f1be76917d2a21925f290bb2736",
|
||||
"delivered": 4489,
|
||||
"superseded": 0,
|
||||
"operating_target_gate_passed": True,
|
||||
},
|
||||
}
|
||||
LOAD_ENVELOPE_PROFILE_SHA256: Final = (
|
||||
"f0c9467d71622085b31f650d89b70ad5107fa49842f1bc828ccc2c83f85628a3"
|
||||
)
|
||||
LOAD_RUNTIME_ARTIFACT_SHA256: Final = (
|
||||
"9929b735fd139369812f2d5c67149f7737cbf3d3f34076b590e9e9672666bf03"
|
||||
)
|
||||
LOAD_RUNNER_SHA256: Final = (
|
||||
"d3ca51b8500b681307d3a9974f57ae72cdbf04267b5bba26e7ffb091ddd9df31"
|
||||
)
|
||||
YOLOX_ID: Final = (
|
||||
"m48s-yolox-all-coco-shadow-7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06"
|
||||
)
|
||||
@@ -115,6 +261,24 @@ def build_m48s_fixed_class_detector_lab(
|
||||
reference_graph_replay_path = reference_graph_replay_root / "manifest.json"
|
||||
reference_graph_replay_worker_path = reference_graph_replay_root / "worker-result.json"
|
||||
reference_graph_replay_frames_path = reference_graph_replay_root / "frames.jsonl"
|
||||
hardening_root = runtime / "reference-graph-replay-results"
|
||||
hardening_paths = {
|
||||
name: {
|
||||
"result": hardening_root / str(definition["directory"]) / "result.json",
|
||||
"frames": hardening_root / str(definition["directory"]) / "frames.jsonl",
|
||||
}
|
||||
for name, definition in RUNTIME_HARDENING_RUNS.items()
|
||||
}
|
||||
load_envelope_profile_path = (
|
||||
repository / "config/perception/m48s-load-envelope-prefetch-v1.json"
|
||||
)
|
||||
load_envelope_paths = {
|
||||
name: runtime
|
||||
/ "load-envelope-prefetch-worker"
|
||||
/ str(definition["run_id"])
|
||||
/ "result.json"
|
||||
for name, definition in LOAD_ENVELOPE_RUNS.items()
|
||||
}
|
||||
yolox_root = runtime / "yolox-all-coco-results" / YOLOX_ID
|
||||
yolox_manifest_path = yolox_root / "manifest.json"
|
||||
yolox_frames_path = yolox_root / "frames.jsonl"
|
||||
@@ -137,6 +301,9 @@ def build_m48s_fixed_class_detector_lab(
|
||||
dfine_path,
|
||||
rf_detr_path,
|
||||
profile_path,
|
||||
load_envelope_profile_path,
|
||||
*load_envelope_paths.values(),
|
||||
*(path for paths in hardening_paths.values() for path in paths.values()),
|
||||
):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise M48SFixedClassDetectorLabError(
|
||||
@@ -150,6 +317,18 @@ def build_m48s_fixed_class_detector_lab(
|
||||
reference_graph_worker = _read_object(reference_graph_worker_path)
|
||||
reference_graph_replay = _read_object(reference_graph_replay_path)
|
||||
reference_graph_replay_worker = _read_object(reference_graph_replay_worker_path)
|
||||
hardening_runs = {
|
||||
name: _read_object(paths["result"]) for name, paths in hardening_paths.items()
|
||||
}
|
||||
load_envelope_profile = _read_object(load_envelope_profile_path)
|
||||
load_envelope_runs = {
|
||||
name: _read_object(path) for name, path in load_envelope_paths.items()
|
||||
}
|
||||
hardening_first_frames = {
|
||||
name: _read_first_jsonl_object(paths["frames"])
|
||||
for name, paths in hardening_paths.items()
|
||||
if name in {"hardened", "prewarmed"}
|
||||
}
|
||||
yolox_manifest = _read_object(yolox_manifest_path)
|
||||
dfine = _read_object(dfine_path)
|
||||
rf_detr = _read_object(rf_detr_path)
|
||||
@@ -169,6 +348,13 @@ def build_m48s_fixed_class_detector_lab(
|
||||
rf_detr=rf_detr,
|
||||
profile=profile,
|
||||
yolox_frames=yolox_frames,
|
||||
hardening_runs=hardening_runs,
|
||||
hardening_first_frames=hardening_first_frames,
|
||||
hardening_paths=hardening_paths,
|
||||
load_envelope_profile=load_envelope_profile,
|
||||
load_envelope_profile_path=load_envelope_profile_path,
|
||||
load_envelope_runs=load_envelope_runs,
|
||||
load_envelope_paths=load_envelope_paths,
|
||||
)
|
||||
|
||||
source_paths = {frame_id: source_root / f"frame-{frame_id}.jpg" for frame_id in FRAME_IDS}
|
||||
@@ -221,15 +407,26 @@ def build_m48s_fixed_class_detector_lab(
|
||||
"reference_graph_document_sha256": sha256_path(reference_graph_path),
|
||||
"reference_graph_worker_sha256": sha256_path(reference_graph_worker_path),
|
||||
"reference_graph_replay_result_id": REFERENCE_GRAPH_REPLAY_ID,
|
||||
"reference_graph_replay_document_sha256": sha256_path(
|
||||
reference_graph_replay_path
|
||||
),
|
||||
"reference_graph_replay_worker_sha256": sha256_path(
|
||||
reference_graph_replay_worker_path
|
||||
),
|
||||
"reference_graph_replay_frames_sha256": sha256_path(
|
||||
reference_graph_replay_frames_path
|
||||
),
|
||||
"reference_graph_replay_document_sha256": sha256_path(reference_graph_replay_path),
|
||||
"reference_graph_replay_worker_sha256": sha256_path(reference_graph_replay_worker_path),
|
||||
"reference_graph_replay_frames_sha256": sha256_path(reference_graph_replay_frames_path),
|
||||
"runtime_hardening": {
|
||||
name: {
|
||||
"result_sha256": sha256_path(paths["result"]),
|
||||
"frames_sha256": sha256_path(paths["frames"]),
|
||||
}
|
||||
for name, paths in hardening_paths.items()
|
||||
},
|
||||
"load_envelope": {
|
||||
"profile_sha256": sha256_path(load_envelope_profile_path),
|
||||
"runs": {
|
||||
name: {
|
||||
"result_sha256": sha256_path(load_envelope_paths[name]),
|
||||
"frames_sha256": run["execution"]["frame_evidence"]["sha256"],
|
||||
}
|
||||
for name, run in load_envelope_runs.items()
|
||||
},
|
||||
},
|
||||
"yolox_result_id": YOLOX_ID,
|
||||
"yolox_document_sha256": sha256_path(yolox_manifest_path),
|
||||
},
|
||||
@@ -238,7 +435,7 @@ def build_m48s_fixed_class_detector_lab(
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||
result_id = RESULT_PREFIX + identity_sha256
|
||||
completed_utc_ns = reference_graph_replay_worker.get("completed_utc_ns")
|
||||
completed_utc_ns = load_envelope_runs["limit-15fps-a3"].get("completed_utc_ns")
|
||||
if not isinstance(completed_utc_ns, int) or isinstance(completed_utc_ns, bool):
|
||||
raise M48SFixedClassDetectorLabError(
|
||||
"complete reference-graph completion time is unavailable"
|
||||
@@ -260,7 +457,11 @@ def build_m48s_fixed_class_detector_lab(
|
||||
load=load,
|
||||
reference_graph=reference_graph,
|
||||
candidates=candidates,
|
||||
hardening_runs=hardening_runs,
|
||||
hardening_first_frames=hardening_first_frames,
|
||||
load_envelope_runs=load_envelope_runs,
|
||||
)
|
||||
envelope = cast(dict[str, Any], metrics["load_envelope"])
|
||||
decision = {
|
||||
"bounded_question_accepted": True,
|
||||
"selected_candidate": "rf-detr",
|
||||
@@ -268,6 +469,13 @@ def build_m48s_fixed_class_detector_lab(
|
||||
"integrated_world_state_gate_evaluated": True,
|
||||
"integrated_world_state_gate_passed": True,
|
||||
"full_replay_visual_published": True,
|
||||
"load_envelope_evaluated": True,
|
||||
"production_rate_repeatability_passed": envelope[
|
||||
"production_rate_repeatability_passed"
|
||||
],
|
||||
"reserve_12_fps_passed": envelope["reserve_12_fps_passed"],
|
||||
"limit_15_fps_passed": envelope["limit_15_fps_passed"],
|
||||
"load_envelope_accepted": envelope["load_envelope_accepted"],
|
||||
"detector_replacement_authorized": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
@@ -283,8 +491,13 @@ def build_m48s_fixed_class_detector_lab(
|
||||
"authority."
|
||||
),
|
||||
(
|
||||
"Eight source frames were superseded by the qualified latest-wins graph; their "
|
||||
"camera/LiDAR source evidence remains visible without invented world state."
|
||||
"The visual timeline is the original qualified semantic replay; runtime-hardening "
|
||||
"metrics come from separately sealed, input-identical replay runs."
|
||||
),
|
||||
(
|
||||
"Three complete repetitions at each of 10, 12 and 15 FPS passed their predeclared "
|
||||
"integrity and operating-target gates after bounded decode prefetch was completed "
|
||||
"before source admission."
|
||||
),
|
||||
]
|
||||
|
||||
@@ -370,6 +583,24 @@ def build_m48s_fixed_class_detector_lab(
|
||||
reference_graph_replay_frames_path,
|
||||
temporary / "reference-graph-replay-frames.jsonl",
|
||||
)
|
||||
for name, paths in hardening_paths.items():
|
||||
shutil.copyfile(paths["result"], temporary / f"runtime-hardening-{name}.json")
|
||||
shutil.copyfile(
|
||||
load_envelope_profile_path,
|
||||
temporary / "runtime-load-envelope-profile.json",
|
||||
)
|
||||
for name, path in load_envelope_paths.items():
|
||||
shutil.copyfile(path, temporary / f"runtime-load-{name}.json")
|
||||
startup_evidence = {
|
||||
"schema_version": RUNTIME_HARDENING_SCHEMA,
|
||||
"source_frames_sha256": {
|
||||
name: sha256_path(hardening_paths[name]["frames"])
|
||||
for name in ("hardened", "prewarmed")
|
||||
},
|
||||
"first_delivered_frames": hardening_first_frames,
|
||||
}
|
||||
startup_path = temporary / "runtime-hardening-startup.json"
|
||||
startup_path.write_bytes(canonical_json(startup_evidence) + b"\n")
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
@@ -379,11 +610,16 @@ def build_m48s_fixed_class_detector_lab(
|
||||
"execution": {
|
||||
"detector_load": load["execution"],
|
||||
"complete_reference_graph": reference_graph["identity"]["evidence"]["execution"],
|
||||
"runtime_hardening": metrics["runtime_hardening"],
|
||||
"load_envelope": metrics["load_envelope"],
|
||||
},
|
||||
"metrics": metrics,
|
||||
"acceptance": {
|
||||
"detector_load": load["checks"],
|
||||
"complete_reference_graph": reference_graph["identity"]["evidence"]["checks"],
|
||||
"load_envelope": {
|
||||
name: run["checks"] for name, run in load_envelope_runs.items()
|
||||
},
|
||||
},
|
||||
"decision": decision,
|
||||
"limitations": limitations,
|
||||
@@ -450,6 +686,13 @@ def _validate_inputs(
|
||||
rf_detr: dict[str, Any],
|
||||
profile: dict[str, Any],
|
||||
yolox_frames: list[dict[str, Any]],
|
||||
hardening_runs: dict[str, dict[str, Any]],
|
||||
hardening_first_frames: dict[str, dict[str, Any]],
|
||||
hardening_paths: dict[str, dict[str, Path]],
|
||||
load_envelope_profile: dict[str, Any],
|
||||
load_envelope_profile_path: Path,
|
||||
load_envelope_runs: dict[str, dict[str, Any]],
|
||||
load_envelope_paths: dict[str, Path],
|
||||
) -> None:
|
||||
decision = deployment.get("decision")
|
||||
graph_identity = reference_graph.get("identity")
|
||||
@@ -460,8 +703,7 @@ def _validate_inputs(
|
||||
replay_artifacts = reference_graph_replay.get("artifacts")
|
||||
replay_frame_summary = (
|
||||
replay_identity.get("evidence", {}).get("frame_summary")
|
||||
if isinstance(replay_identity, dict)
|
||||
and isinstance(replay_identity.get("evidence"), dict)
|
||||
if isinstance(replay_identity, dict) and isinstance(replay_identity.get("evidence"), dict)
|
||||
else None
|
||||
)
|
||||
if (
|
||||
@@ -539,6 +781,194 @@ def _validate_inputs(
|
||||
authority = document.get("authority")
|
||||
if authority is not None and authority != false_authority():
|
||||
raise M48SFixedClassDetectorLabError("sealed M4.8S evidence gained authority")
|
||||
for name, definition in RUNTIME_HARDENING_RUNS.items():
|
||||
run = hardening_runs.get(name)
|
||||
paths = hardening_paths.get(name)
|
||||
if not isinstance(run, dict) or not isinstance(paths, dict):
|
||||
raise M48SFixedClassDetectorLabError("runtime-hardening evidence is incomplete")
|
||||
checks = run.get("checks")
|
||||
execution = run.get("execution")
|
||||
identity = run.get("identity")
|
||||
frame_evidence = execution.get("frame_evidence") if isinstance(execution, dict) else None
|
||||
if (
|
||||
run.get("schema_version") != definition["schema_version"]
|
||||
or run.get("completed") is not True
|
||||
or run.get("integrated_runtime_gate_passed") is not True
|
||||
or run.get("production_accepted") is not False
|
||||
or run.get("authority") != false_authority()
|
||||
or not isinstance(checks, dict)
|
||||
or not checks
|
||||
or not all(value is True for value in checks.values())
|
||||
or not isinstance(identity, dict)
|
||||
or identity.get("graph_id") != "reference-perception-graph/v2"
|
||||
or identity.get("detector_provider_id")
|
||||
!= "triton-rf-detr-large-coco-risk-fp16-shadow/v0"
|
||||
or identity.get("worker_id") != "worker-006"
|
||||
or not isinstance(execution, dict)
|
||||
or execution.get("admitted_frames") != definition["admitted_count"]
|
||||
or not isinstance(frame_evidence, dict)
|
||||
or frame_evidence.get("row_count") != definition["frame_count"]
|
||||
or frame_evidence.get("sha256") != definition["frames_sha256"]
|
||||
or sha256_path(paths["result"]) != definition["result_sha256"]
|
||||
or sha256_path(paths["frames"]) != definition["frames_sha256"]
|
||||
):
|
||||
raise M48SFixedClassDetectorLabError(
|
||||
f"sealed runtime-hardening evidence changed: {name}"
|
||||
)
|
||||
for name in ("hardened", "prewarmed"):
|
||||
frame = hardening_first_frames.get(name)
|
||||
timing = frame.get("pipeline_timing") if isinstance(frame, dict) else None
|
||||
detector = timing.get("detector") if isinstance(timing, dict) else None
|
||||
if (
|
||||
not isinstance(frame, dict)
|
||||
or frame.get("schema_version") != "missioncore.m48s-reference-graph-frame-evidence/v1"
|
||||
or not isinstance(timing, dict)
|
||||
or timing.get("sequence") != 0
|
||||
or not isinstance(timing.get("graph_admission_to_delivery_ns"), int)
|
||||
or not isinstance(detector, dict)
|
||||
or detector.get("sequence") != 0
|
||||
or not isinstance(detector.get("total_duration_ns"), int)
|
||||
):
|
||||
raise M48SFixedClassDetectorLabError(
|
||||
f"runtime-hardening startup evidence changed: {name}"
|
||||
)
|
||||
_validate_load_envelope(
|
||||
profile=load_envelope_profile,
|
||||
profile_path=load_envelope_profile_path,
|
||||
runs=load_envelope_runs,
|
||||
paths=load_envelope_paths,
|
||||
)
|
||||
|
||||
|
||||
def _validate_load_envelope(
|
||||
*,
|
||||
profile: dict[str, Any],
|
||||
profile_path: Path,
|
||||
runs: dict[str, dict[str, Any]],
|
||||
paths: dict[str, Path],
|
||||
) -> None:
|
||||
scenarios = profile.get("scenarios")
|
||||
if (
|
||||
profile.get("schema_version") != "missioncore.m48s-load-envelope-profile/v2"
|
||||
or profile.get("profile_id")
|
||||
!= "m48s-rf-detr-reference-graph-load-envelope-prefetch/v1"
|
||||
or sha256_path(profile_path) != LOAD_ENVELOPE_PROFILE_SHA256
|
||||
or not isinstance(scenarios, list)
|
||||
or len(scenarios) != 3
|
||||
or profile.get("series_control", {}).get("repeat_count_per_rate") != 3
|
||||
):
|
||||
raise M48SFixedClassDetectorLabError("load-envelope profile identity changed")
|
||||
scenarios_by_id = {
|
||||
scenario.get("id"): scenario for scenario in scenarios if isinstance(scenario, dict)
|
||||
}
|
||||
expected_scenario_ids = {
|
||||
str(definition["scenario_id"]) for definition in LOAD_ENVELOPE_RUNS.values()
|
||||
}
|
||||
if set(scenarios_by_id) != expected_scenario_ids:
|
||||
raise M48SFixedClassDetectorLabError("load-envelope scenarios changed")
|
||||
declared_run_ids = {
|
||||
run_id
|
||||
for scenario in scenarios_by_id.values()
|
||||
for run_id in cast(list[object], scenario.get("run_ids", []))
|
||||
}
|
||||
expected_run_ids = {
|
||||
str(definition["run_id"]) for definition in LOAD_ENVELOPE_RUNS.values()
|
||||
}
|
||||
if declared_run_ids != expected_run_ids:
|
||||
raise M48SFixedClassDetectorLabError("load-envelope run series changed")
|
||||
shared_inputs: dict[str, object] | None = None
|
||||
for name, definition in LOAD_ENVELOPE_RUNS.items():
|
||||
run = runs.get(name)
|
||||
path = paths.get(name)
|
||||
scenario = scenarios_by_id[str(definition["scenario_id"])]
|
||||
if not isinstance(run, dict) or not isinstance(path, Path):
|
||||
raise M48SFixedClassDetectorLabError("load-envelope evidence is incomplete")
|
||||
source = run.get("source")
|
||||
identity = run.get("identity")
|
||||
execution = run.get("execution")
|
||||
integrity_checks = run.get("integrity_checks")
|
||||
thresholds = run.get("predeclared_thresholds")
|
||||
frame_evidence = execution.get("frame_evidence") if isinstance(execution, dict) else None
|
||||
terminal = execution.get("terminal_outcomes") if isinstance(execution, dict) else None
|
||||
pipeline = run.get("metrics", {}).get("pipeline_timing")
|
||||
decode_by_phase = (
|
||||
pipeline.get("decode_duration_by_phase_ms") if isinstance(pipeline, dict) else None
|
||||
)
|
||||
pacing = (
|
||||
pipeline.get("delivered_source_pacing_lateness_ms")
|
||||
if isinstance(pipeline, dict)
|
||||
else None
|
||||
)
|
||||
loops = execution.get("loops") if isinstance(execution, dict) else None
|
||||
prefetch = (
|
||||
loops[0].get("source_prefetch")
|
||||
if isinstance(loops, list) and len(loops) == 1 and isinstance(loops[0], dict)
|
||||
else None
|
||||
)
|
||||
if (
|
||||
run.get("schema_version") != "missioncore.m48s-reference-graph-shadow-load/v5"
|
||||
or run.get("completed") is not True
|
||||
or run.get("production_accepted") is not False
|
||||
or run.get("authority") != false_authority()
|
||||
or run.get("evidence_integrity_gate_passed") is not True
|
||||
or run.get("operating_target_gate_passed")
|
||||
is not definition["operating_target_gate_passed"]
|
||||
or not isinstance(integrity_checks, dict)
|
||||
or not integrity_checks
|
||||
or not all(value is True for value in integrity_checks.values())
|
||||
or not isinstance(source, dict)
|
||||
or source.get("source_id") != "RAVNOVES00"
|
||||
or source.get("pacing_contract")
|
||||
!= "wall-clock-scaled-source-timestamps-immutable/v1"
|
||||
or source.get("requested_rate_hz") != definition["source_rate_hz"]
|
||||
or not isinstance(identity, dict)
|
||||
or identity.get("worker_id") != "worker-006"
|
||||
or identity.get("graph_id") != "reference-perception-graph/v2"
|
||||
or identity.get("detector_provider_id")
|
||||
!= "triton-rf-detr-large-coco-risk-fp16-shadow/v0"
|
||||
or identity.get("runtime_artifact_sha256") != LOAD_RUNTIME_ARTIFACT_SHA256
|
||||
or identity.get("runner_sha256") != LOAD_RUNNER_SHA256
|
||||
or not isinstance(execution, dict)
|
||||
or execution.get("load_purpose") != definition["load_purpose"]
|
||||
or execution.get("requested_source_rate_hz") != definition["source_rate_hz"]
|
||||
or execution.get("source_timestamps_preserved") is not True
|
||||
or execution.get("admitted_frames") != 4489
|
||||
or execution.get("delivered_world_states") != definition["delivered"]
|
||||
or not isinstance(terminal, dict)
|
||||
or terminal.get("delivered") != definition["delivered"]
|
||||
or terminal.get("superseded", 0) != definition["superseded"]
|
||||
or sum(terminal.values()) != 4489
|
||||
or not isinstance(frame_evidence, dict)
|
||||
or frame_evidence.get("row_count") != definition["delivered"]
|
||||
or frame_evidence.get("sha256") != definition["frames_sha256"]
|
||||
or not isinstance(pipeline, dict)
|
||||
or pipeline.get("additional_inference_passes") != 0
|
||||
or not isinstance(decode_by_phase, dict)
|
||||
or set(decode_by_phase) != {"preadmission", "hot-loop"}
|
||||
or any(
|
||||
not isinstance(decode_by_phase.get(phase), dict)
|
||||
for phase in ("preadmission", "hot-loop")
|
||||
)
|
||||
or not isinstance(pacing, dict)
|
||||
or not isinstance(prefetch, dict)
|
||||
or prefetch.get("capacity_frames") != 64
|
||||
or prefetch.get("ready_frames") != 64
|
||||
or prefetch.get("buffered_frames") != 64
|
||||
or not isinstance(prefetch.get("preparation_duration_ns"), int)
|
||||
or thresholds != scenario.get("thresholds")
|
||||
or definition["run_id"] not in scenario.get("run_ids", [])
|
||||
or sha256_path(path) != definition["result_sha256"]
|
||||
):
|
||||
raise M48SFixedClassDetectorLabError(
|
||||
f"sealed load-envelope evidence changed: {name}"
|
||||
)
|
||||
inputs = identity.get("inputs")
|
||||
if not isinstance(inputs, dict):
|
||||
raise M48SFixedClassDetectorLabError("load-envelope input identity is unavailable")
|
||||
if shared_inputs is None:
|
||||
shared_inputs = inputs
|
||||
elif inputs != shared_inputs:
|
||||
raise M48SFixedClassDetectorLabError("load-envelope runs changed graph inputs")
|
||||
|
||||
|
||||
def _method(
|
||||
@@ -674,6 +1104,9 @@ def _metrics(
|
||||
load: dict[str, Any],
|
||||
reference_graph: dict[str, Any],
|
||||
candidates: list[dict[str, object]],
|
||||
hardening_runs: dict[str, dict[str, Any]],
|
||||
hardening_first_frames: dict[str, dict[str, Any]],
|
||||
load_envelope_runs: dict[str, dict[str, Any]],
|
||||
) -> dict[str, object]:
|
||||
graph_evidence = reference_graph["identity"]["evidence"]
|
||||
graph_execution = graph_evidence["execution"]
|
||||
@@ -739,6 +1172,223 @@ def _metrics(
|
||||
for key in ("failed", "stale", "rejected", "unavailable")
|
||||
),
|
||||
},
|
||||
"runtime_hardening": _runtime_hardening_metrics(
|
||||
runs=hardening_runs,
|
||||
first_frames=hardening_first_frames,
|
||||
),
|
||||
"load_envelope": _load_envelope_metrics(load_envelope_runs),
|
||||
}
|
||||
|
||||
|
||||
def _load_envelope_metrics(runs: dict[str, dict[str, Any]]) -> dict[str, object]:
|
||||
scenarios: list[dict[str, object]] = []
|
||||
scenario_order = ("production-10fps", "reserve-12fps", "limit-15fps")
|
||||
for scenario_id in scenario_order:
|
||||
definitions = sorted(
|
||||
(
|
||||
(name, definition)
|
||||
for name, definition in LOAD_ENVELOPE_RUNS.items()
|
||||
if definition["scenario_id"] == scenario_id
|
||||
),
|
||||
key=lambda item: int(item[1]["repetition"]),
|
||||
)
|
||||
repetitions: list[dict[str, object]] = []
|
||||
queue_high_watermarks: dict[str, int] = {}
|
||||
for name, definition in definitions:
|
||||
run = runs[name]
|
||||
execution = run["execution"]
|
||||
metrics = run["metrics"]
|
||||
completion = metrics["world_state_completion_age_ms"]
|
||||
pipeline = metrics["pipeline_timing"]
|
||||
decode = pipeline["decode_duration_by_phase_ms"]
|
||||
pacing = pipeline["delivered_source_pacing_lateness_ms"]
|
||||
terminal = execution["terminal_outcomes"]
|
||||
prefetch = execution["loops"][0]["source_prefetch"]
|
||||
for queue, depth in execution["queue_high_watermarks"].items():
|
||||
queue_high_watermarks[queue] = max(
|
||||
queue_high_watermarks.get(queue, 0), int(depth)
|
||||
)
|
||||
repetitions.append(
|
||||
{
|
||||
"run_id": definition["run_id"],
|
||||
"repetition": definition["repetition"],
|
||||
"delivered_world_states": execution["delivered_world_states"],
|
||||
"superseded_frames": terminal.get("superseded", 0),
|
||||
"delivery_ratio": execution["delivery_ratio"],
|
||||
"effective_world_state_fps": execution["effective_world_state_fps"],
|
||||
"world_state_completion_age_p95_ms": completion["p95"],
|
||||
"hot_loop_decode_p95_ms": decode["hot-loop"]["p95"],
|
||||
"hot_loop_decode_maximum_ms": decode["hot-loop"]["maximum"],
|
||||
"preadmission_decode_maximum_ms": decode["preadmission"]["maximum"],
|
||||
"source_pacing_lateness_maximum_ms": pacing["maximum"],
|
||||
"source_prefetch_preparation_ms": (
|
||||
prefetch["preparation_duration_ns"] / 1_000_000
|
||||
),
|
||||
"operating_target_gate_passed": run["operating_target_gate_passed"],
|
||||
"result_sha256": definition["result_sha256"],
|
||||
"frame_evidence_sha256": execution["frame_evidence"]["sha256"],
|
||||
}
|
||||
)
|
||||
first_name = definitions[0][0]
|
||||
first_run = runs[first_name]
|
||||
run_values = [runs[name] for name, _definition in definitions]
|
||||
executions = [run["execution"] for run in run_values]
|
||||
run_metrics = [run["metrics"] for run in run_values]
|
||||
pipelines = [metrics["pipeline_timing"] for metrics in run_metrics]
|
||||
completions = [metrics["world_state_completion_age_ms"] for metrics in run_metrics]
|
||||
gpus = [metrics["gpu"] for metrics in run_metrics]
|
||||
all_integrity = all(run["evidence_integrity_gate_passed"] for run in run_values)
|
||||
all_targets = all(run["operating_target_gate_passed"] for run in run_values)
|
||||
scenarios.append(
|
||||
{
|
||||
"id": scenario_id,
|
||||
"load_purpose": first_run["execution"]["load_purpose"],
|
||||
"requested_source_rate_hz": first_run["execution"][
|
||||
"requested_source_rate_hz"
|
||||
],
|
||||
"repeat_count": len(repetitions),
|
||||
"all_repetitions_passed": all_integrity and all_targets,
|
||||
"source_frames_admitted": min(
|
||||
execution["admitted_frames"] for execution in executions
|
||||
),
|
||||
"delivered_world_states": min(
|
||||
execution["delivered_world_states"] for execution in executions
|
||||
),
|
||||
"superseded_frames": max(
|
||||
execution["terminal_outcomes"].get("superseded", 0)
|
||||
for execution in executions
|
||||
),
|
||||
"delivery_ratio": min(execution["delivery_ratio"] for execution in executions),
|
||||
"effective_world_state_fps": min(
|
||||
execution["effective_world_state_fps"] for execution in executions
|
||||
),
|
||||
"world_state_completion_age_p95_ms": max(
|
||||
completion["p95"] for completion in completions
|
||||
),
|
||||
"world_state_completion_age_p99_ms": max(
|
||||
completion["p99"] for completion in completions
|
||||
),
|
||||
"world_state_completion_age_maximum_ms": max(
|
||||
completion["maximum"] for completion in completions
|
||||
),
|
||||
"hot_loop_decode_p95_ms": max(
|
||||
pipeline["decode_duration_by_phase_ms"]["hot-loop"]["p95"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"hot_loop_decode_maximum_ms": max(
|
||||
pipeline["decode_duration_by_phase_ms"]["hot-loop"]["maximum"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"preadmission_decode_maximum_ms": max(
|
||||
pipeline["decode_duration_by_phase_ms"]["preadmission"]["maximum"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"source_prefetch_preparation_maximum_ms": max(
|
||||
repetition["source_prefetch_preparation_ms"]
|
||||
for repetition in repetitions
|
||||
),
|
||||
"source_pacing_lateness_p95_ms": max(
|
||||
pipeline["delivered_source_pacing_lateness_ms"]["p95"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"source_pacing_lateness_maximum_ms": max(
|
||||
pipeline["delivered_source_pacing_lateness_ms"]["maximum"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"detector_p95_ms": max(
|
||||
pipeline["detector_ms"]["total"]["p95"] for pipeline in pipelines
|
||||
),
|
||||
"detector_maximum_ms": max(
|
||||
pipeline["detector_ms"]["total"]["maximum"]
|
||||
for pipeline in pipelines
|
||||
),
|
||||
"gpu_utilization_mean_percent": max(
|
||||
gpu["gpu_utilization_percent"]["mean"] for gpu in gpus
|
||||
),
|
||||
"gpu_utilization_maximum_percent": max(
|
||||
gpu["gpu_utilization_percent"]["maximum"] for gpu in gpus
|
||||
),
|
||||
"gpu_memory_maximum_mib": max(
|
||||
gpu["gpu_memory_used_mib"]["maximum"] for gpu in gpus
|
||||
),
|
||||
"process_peak_rss_mib": max(
|
||||
metrics["process_peak_rss_after_mib"] for metrics in run_metrics
|
||||
),
|
||||
"queue_high_watermarks": queue_high_watermarks,
|
||||
"queue_capacity": 2,
|
||||
"additional_inference_passes": max(
|
||||
pipeline["additional_inference_passes"] for pipeline in pipelines
|
||||
),
|
||||
"integrity_gate_passed": all_integrity,
|
||||
"operating_target_gate_passed": all_targets,
|
||||
"thresholds": first_run["predeclared_thresholds"],
|
||||
"repetitions": repetitions,
|
||||
}
|
||||
)
|
||||
production_passed = cast(bool, scenarios[0]["all_repetitions_passed"])
|
||||
reserve_passed = cast(bool, scenarios[1]["all_repetitions_passed"])
|
||||
limit_passed = cast(bool, scenarios[2]["all_repetitions_passed"])
|
||||
return {
|
||||
"schema_version": LOAD_ENVELOPE_SCHEMA,
|
||||
"repeat_count_per_rate": 3,
|
||||
"production_rate_repeatability_passed": production_passed,
|
||||
"reserve_12_fps_passed": reserve_passed,
|
||||
"limit_15_fps_passed": limit_passed,
|
||||
"load_envelope_accepted": production_passed and reserve_passed,
|
||||
"compute_capacity_at_least_fps": (
|
||||
15.0 if limit_passed else 12.0 if reserve_passed else 10.0
|
||||
),
|
||||
"bottleneck_interpretation": (
|
||||
"cold-video-decode-isolated-before-admission-no-steady-gpu-saturation"
|
||||
),
|
||||
"scenarios": scenarios,
|
||||
}
|
||||
|
||||
|
||||
def _runtime_hardening_metrics(
|
||||
*,
|
||||
runs: dict[str, dict[str, Any]],
|
||||
first_frames: dict[str, dict[str, Any]],
|
||||
) -> dict[str, object]:
|
||||
def full_run(name: str) -> dict[str, object]:
|
||||
run = runs[name]
|
||||
execution = run["execution"]
|
||||
completion = run["metrics"]["world_state_completion_age_ms"]
|
||||
pipeline = run["metrics"]["pipeline_timing"]
|
||||
terminal = execution["terminal_outcomes"]
|
||||
return {
|
||||
"source_frames_admitted": execution["admitted_frames"],
|
||||
"delivered_world_states": execution["delivered_world_states"],
|
||||
"superseded_frames": terminal.get("superseded", 0),
|
||||
"effective_world_state_fps": execution["effective_world_state_fps"],
|
||||
"world_state_completion_age_p95_ms": completion["p95"],
|
||||
"world_state_completion_age_p99_ms": completion["p99"],
|
||||
"world_state_completion_age_maximum_ms": completion["maximum"],
|
||||
"rolling_maximum_ms": pipeline["provider_ms"]["rolling"]["maximum"],
|
||||
"geometry_maximum_ms": pipeline["provider_ms"]["geometry"]["maximum"],
|
||||
"additional_inference_passes": pipeline["additional_inference_passes"],
|
||||
}
|
||||
|
||||
def startup_frame(name: str) -> dict[str, float]:
|
||||
timing = first_frames[name]["pipeline_timing"]
|
||||
return {
|
||||
"detector_ms": timing["detector"]["total_duration_ns"] / 1_000_000,
|
||||
"world_state_ms": timing["graph_admission_to_delivery_ns"] / 1_000_000,
|
||||
}
|
||||
|
||||
prewarmed_loop = runs["prewarmed"]["execution"]["loops"][0]
|
||||
warmup = prewarmed_loop["detector_warmup"]
|
||||
return {
|
||||
"schema_version": RUNTIME_HARDENING_SCHEMA,
|
||||
"baseline": full_run("baseline"),
|
||||
"hardened": full_run("hardened"),
|
||||
"startup": {
|
||||
"baseline": startup_frame("hardened"),
|
||||
"prewarmed": startup_frame("prewarmed"),
|
||||
"prewarm_duration_ms": warmup["total_duration_ns"] / 1_000_000,
|
||||
"prewarm_inference_passes": warmup["inference_passes"],
|
||||
"validation_frames": runs["prewarmed"]["execution"]["admitted_frames"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -843,6 +1493,17 @@ def _artifact_manifest(root: Path) -> list[dict[str, object]]:
|
||||
media_type = "application/x-ndjson"
|
||||
schema_version = "missioncore.m48s-reference-graph-frame-evidence/v0"
|
||||
role = "visual-evidence-full-replay-world-state"
|
||||
elif relative.startswith("runtime-hardening-"):
|
||||
schema_version = (
|
||||
RUNTIME_HARDENING_SCHEMA if relative == "runtime-hardening-startup.json" else None
|
||||
)
|
||||
role = "upstream-runtime-hardening-evidence"
|
||||
elif relative == "runtime-load-envelope-profile.json":
|
||||
schema_version = "missioncore.m48s-load-envelope-profile/v2"
|
||||
role = "predeclared-load-envelope-profile"
|
||||
elif relative.startswith("runtime-load-"):
|
||||
schema_version = "missioncore.m48s-reference-graph-shadow-load/v5"
|
||||
role = "upstream-load-envelope-evidence"
|
||||
artifacts.append(
|
||||
{
|
||||
"role": role,
|
||||
@@ -866,6 +1527,18 @@ def _read_object(path: Path) -> dict[str, Any]:
|
||||
return value
|
||||
|
||||
|
||||
def _read_first_jsonl_object(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as stream:
|
||||
line = stream.readline()
|
||||
value = json.loads(line)
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise M48SFixedClassDetectorLabError(f"invalid first-frame evidence: {path.name}") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise M48SFixedClassDetectorLabError(f"first-frame evidence must be an object: {path.name}")
|
||||
return value
|
||||
|
||||
|
||||
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
try:
|
||||
rows = [json.loads(line) for line in path.read_text("utf-8").splitlines() if line]
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Versioned launchd declaration for the canonical local Mission Core service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import plistlib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
MISSION_CORE_LAUNCH_AGENT_LABEL: Final = "com.nodedc.mission-core.local"
|
||||
MISSION_CORE_LAUNCH_AGENT_SCHEMA: Final = "missioncore.local-launch-agent-plan/v1"
|
||||
|
||||
|
||||
class MissionCoreLaunchAgentError(RuntimeError):
|
||||
"""The local launch agent cannot be planned without weakening its boundary."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MissionCoreLaunchAgentPlan:
|
||||
agent_path: Path
|
||||
current_sha256: str
|
||||
desired_sha256: str
|
||||
current_program_arguments: tuple[str, ...]
|
||||
desired_program_arguments: tuple[str, ...]
|
||||
desired_payload: bytes
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": MISSION_CORE_LAUNCH_AGENT_SCHEMA,
|
||||
"label": MISSION_CORE_LAUNCH_AGENT_LABEL,
|
||||
"agent_path": str(self.agent_path),
|
||||
"current_sha256": self.current_sha256,
|
||||
"desired_sha256": self.desired_sha256,
|
||||
"current_program_arguments": list(self.current_program_arguments),
|
||||
"desired_program_arguments": list(self.desired_program_arguments),
|
||||
"changes": {
|
||||
"dependency_sync_disabled": "--no-sync"
|
||||
in self.desired_program_arguments,
|
||||
"self_health_watchdog": True,
|
||||
"bounded_launchd_exit_timeout_seconds": 20,
|
||||
"keep_alive": True,
|
||||
"process_group_owned": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def plan_mission_core_launch_agent(
|
||||
*,
|
||||
repository_root: Path,
|
||||
agent_path: Path,
|
||||
) -> MissionCoreLaunchAgentPlan:
|
||||
repository = repository_root.expanduser().resolve(strict=True)
|
||||
path = agent_path.expanduser().absolute()
|
||||
current_payload = _read_private_regular_file(path)
|
||||
try:
|
||||
current = plistlib.loads(current_payload)
|
||||
except plistlib.InvalidFileException as exc:
|
||||
raise MissionCoreLaunchAgentError("current Mission Core launch agent is invalid") from exc
|
||||
if not isinstance(current, dict) or current.get("Label") != MISSION_CORE_LAUNCH_AGENT_LABEL:
|
||||
raise MissionCoreLaunchAgentError("current launch agent identity changed")
|
||||
current_arguments = _program_arguments(current)
|
||||
current_working_directory = current.get("WorkingDirectory")
|
||||
if current_working_directory != str(repository):
|
||||
raise MissionCoreLaunchAgentError("current launch agent targets another repository")
|
||||
environment = current.get("EnvironmentVariables")
|
||||
if not isinstance(environment, dict) or any(
|
||||
not isinstance(key, str) or not isinstance(value, str)
|
||||
for key, value in environment.items()
|
||||
):
|
||||
raise MissionCoreLaunchAgentError("current launch agent environment is invalid")
|
||||
# A LaunchAgent started directly from this repository's venv is denied
|
||||
# access to ``.venv/pyvenv.cfg`` by macOS privacy controls because the
|
||||
# checkout is below Downloads. The Homebrew uv launcher is already the
|
||||
# accepted local execution boundary. ``--no-sync`` keeps launch startup
|
||||
# deterministic and prevents dependency mutation during recovery.
|
||||
uv_entrypoint = Path(current_arguments[0])
|
||||
if (
|
||||
not uv_entrypoint.is_absolute()
|
||||
or uv_entrypoint.name != "uv"
|
||||
or not uv_entrypoint.exists()
|
||||
):
|
||||
raise MissionCoreLaunchAgentError("Mission Core uv entrypoint is unavailable")
|
||||
desired_environment = dict(environment)
|
||||
desired_environment["MISSIONCORE_SERVICE_WATCHDOG"] = "1"
|
||||
log_path = repository / ".runtime/mission-core/k1link-serve-launchd.log"
|
||||
desired: dict[str, object] = {
|
||||
"Label": MISSION_CORE_LAUNCH_AGENT_LABEL,
|
||||
"ProgramArguments": [
|
||||
str(uv_entrypoint),
|
||||
"run",
|
||||
"--no-sync",
|
||||
"k1link",
|
||||
"serve",
|
||||
],
|
||||
"WorkingDirectory": str(repository),
|
||||
"EnvironmentVariables": desired_environment,
|
||||
"KeepAlive": True,
|
||||
"RunAtLoad": True,
|
||||
"AbandonProcessGroup": False,
|
||||
"ProcessType": "Background",
|
||||
"ThrottleInterval": 5,
|
||||
"ExitTimeOut": 20,
|
||||
"StandardOutPath": str(log_path),
|
||||
"StandardErrorPath": str(log_path),
|
||||
}
|
||||
desired_payload = plistlib.dumps(desired, fmt=plistlib.FMT_XML, sort_keys=True)
|
||||
return MissionCoreLaunchAgentPlan(
|
||||
agent_path=path,
|
||||
current_sha256=_sha256(current_payload),
|
||||
desired_sha256=_sha256(desired_payload),
|
||||
current_program_arguments=current_arguments,
|
||||
desired_program_arguments=tuple(desired["ProgramArguments"]),
|
||||
desired_payload=desired_payload,
|
||||
)
|
||||
|
||||
|
||||
def _program_arguments(document: dict[str, object]) -> tuple[str, ...]:
|
||||
value = document.get("ProgramArguments")
|
||||
if not isinstance(value, list) or not value or any(not isinstance(item, str) for item in value):
|
||||
raise MissionCoreLaunchAgentError("launch agent program arguments are invalid")
|
||||
return tuple(value)
|
||||
|
||||
|
||||
def _read_private_regular_file(path: Path) -> bytes:
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise MissionCoreLaunchAgentError("Mission Core launch agent is unavailable")
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def _sha256(payload: bytes) -> str:
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
|
||||
@@ -32,10 +32,13 @@ from .providers import (
|
||||
)
|
||||
from .recorded_source import (
|
||||
DecodedRecordedSource,
|
||||
DecodePrefetchSnapshot,
|
||||
DecodeTimingObserver,
|
||||
PrefetchedRecordedImageDecoder,
|
||||
PyAvRecordedImageDecoder,
|
||||
RecordedRavnoves00Source,
|
||||
ReplayPacing,
|
||||
SourcePacingObserver,
|
||||
)
|
||||
from .reference_graph_runtime import ReferenceGraphRuntimePaths
|
||||
from .rf_detr_object_detector import (
|
||||
@@ -64,6 +67,8 @@ class M48sReferenceGraphRuntime:
|
||||
|
||||
graph: ReferencePerceptionGraphV2
|
||||
inference_backend: TritonRfDetrHttpInferenceBackend
|
||||
source_prefetch: PrefetchedRecordedImageDecoder
|
||||
_preparation_stop_event: Event = field(default_factory=Event)
|
||||
|
||||
def warm_up_detector(self) -> DetectorWarmupSnapshot:
|
||||
detector = self.graph.detector
|
||||
@@ -71,8 +76,18 @@ class M48sReferenceGraphRuntime:
|
||||
raise M48sReferenceGraphRuntimeError("RF-DETR runtime detector changed before warmup")
|
||||
return detector.warm_up()
|
||||
|
||||
def prepare_source(self) -> DecodePrefetchSnapshot:
|
||||
return self.source_prefetch.prepare(self._preparation_stop_event)
|
||||
|
||||
def mark_source_admission_started(self) -> None:
|
||||
self.source_prefetch.mark_admission_started()
|
||||
|
||||
def close(self) -> None:
|
||||
self.inference_backend.close()
|
||||
self._preparation_stop_event.set()
|
||||
try:
|
||||
self.source_prefetch.close()
|
||||
finally:
|
||||
self.inference_backend.close()
|
||||
|
||||
def __enter__(self) -> M48sReferenceGraphRuntime:
|
||||
return self
|
||||
@@ -90,8 +105,12 @@ def build_m48s_reference_graph_runtime(
|
||||
delivery_observer: Callable[[DeliveredFrame, int], None] | None = None,
|
||||
delivery_evidence_observer: DeliveryEvidenceObserver | None = None,
|
||||
decode_timing_observer: DecodeTimingObserver | None = None,
|
||||
source_pacing_observer: SourcePacingObserver | None = None,
|
||||
detector_timing_observer: DetectorTimingObserver | None = None,
|
||||
maximum_frames: int | None = None,
|
||||
source_rate_hz: float | None = None,
|
||||
source_prefetch_capacity_frames: int = 64,
|
||||
source_prefetch_ready_frames: int = 64,
|
||||
) -> M48sReferenceGraphRuntime:
|
||||
"""Instantiate the complete graph with only its detector pin replaced."""
|
||||
|
||||
@@ -116,6 +135,12 @@ def build_m48s_reference_graph_runtime(
|
||||
|
||||
if maximum_frames is not None and maximum_frames < 1:
|
||||
raise M48sReferenceGraphRuntimeError("maximum frame count must be positive")
|
||||
prefetch = PrefetchedRecordedImageDecoder(
|
||||
PyAvRecordedImageDecoder(paths.video),
|
||||
capacity_frames=source_prefetch_capacity_frames,
|
||||
ready_frames=source_prefetch_ready_frames,
|
||||
timing_observer=decode_timing_observer,
|
||||
)
|
||||
source: SourceProvider = DecodedRecordedSource(
|
||||
source=RecordedRavnoves00Source(
|
||||
camera_index_path=paths.camera_index,
|
||||
@@ -125,9 +150,14 @@ def build_m48s_reference_graph_runtime(
|
||||
if run_mode is GraphRunMode.SOURCE_PACED_LATEST_WINS
|
||||
else ReplayPacing.UNCAPPED
|
||||
),
|
||||
target_rate_hz=(
|
||||
source_rate_hz
|
||||
if run_mode is GraphRunMode.SOURCE_PACED_LATEST_WINS
|
||||
else None
|
||||
),
|
||||
pacing_observer=source_pacing_observer,
|
||||
),
|
||||
decoder=PyAvRecordedImageDecoder(paths.video),
|
||||
timing_observer=decode_timing_observer,
|
||||
decoder=prefetch,
|
||||
)
|
||||
if maximum_frames is not None:
|
||||
source = _LimitedSource(source, maximum_frames)
|
||||
@@ -169,9 +199,16 @@ def build_m48s_reference_graph_runtime(
|
||||
delivery_evidence_observer=delivery_evidence_observer,
|
||||
)
|
||||
except Exception:
|
||||
backend.close()
|
||||
try:
|
||||
prefetch.close()
|
||||
finally:
|
||||
backend.close()
|
||||
raise
|
||||
return M48sReferenceGraphRuntime(graph=graph, inference_backend=backend)
|
||||
return M48sReferenceGraphRuntime(
|
||||
graph=graph,
|
||||
inference_backend=backend,
|
||||
source_prefetch=prefetch,
|
||||
)
|
||||
|
||||
|
||||
class _LimitedSource:
|
||||
|
||||
@@ -10,7 +10,8 @@ from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass, replace
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from queue import Empty, Full, Queue
|
||||
from threading import Event, Lock, Thread
|
||||
from typing import Any, Final, Protocol, cast
|
||||
|
||||
import numpy as np
|
||||
@@ -53,6 +54,11 @@ class ReplayPacing(StrEnum):
|
||||
UNCAPPED = "uncapped"
|
||||
|
||||
|
||||
class DecodePhase(StrEnum):
|
||||
PREADMISSION = "preadmission"
|
||||
HOT_LOOP = "hot-loop"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecordedFrameReference:
|
||||
"""Opaque reference passed to a provider without decoding sensor data."""
|
||||
@@ -89,6 +95,7 @@ WaitFunction = Callable[[Event, float], bool]
|
||||
class DecodedFrameTiming:
|
||||
sequence: int
|
||||
duration_ns: int
|
||||
phase: DecodePhase = DecodePhase.HOT_LOOP
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.sequence < 0 or self.duration_ns < 0:
|
||||
@@ -98,8 +105,49 @@ class DecodedFrameTiming:
|
||||
DecodeTimingObserver = Callable[[DecodedFrameTiming], None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SourcePacingTiming:
|
||||
sequence: int
|
||||
scheduled_monotonic_ns: int
|
||||
emitted_monotonic_ns: int
|
||||
lateness_ns: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if (
|
||||
self.sequence < 0
|
||||
or self.scheduled_monotonic_ns < 0
|
||||
or self.emitted_monotonic_ns < 0
|
||||
or self.lateness_ns < 0
|
||||
):
|
||||
raise RecordedSourceError("source pacing timing must be nonnegative")
|
||||
|
||||
|
||||
SourcePacingObserver = Callable[[SourcePacingTiming], None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DecodePrefetchSnapshot:
|
||||
capacity_frames: int
|
||||
ready_frames: int
|
||||
buffered_frames: int
|
||||
preparation_duration_ns: int
|
||||
producer_alive: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _DecodeFailure:
|
||||
error: BaseException
|
||||
|
||||
|
||||
_DECODE_END: Final = object()
|
||||
|
||||
|
||||
class RecordedRavnoves00Source:
|
||||
"""Emit the admitted synchronized source timeline at 1.0x or uncapped speed."""
|
||||
"""Emit the admitted timeline at its recorded or an explicit replay rate.
|
||||
|
||||
A target rate changes wall-clock pacing only. Immutable sensor timestamps stay
|
||||
untouched so accelerated load measurements cannot silently change scene motion.
|
||||
"""
|
||||
|
||||
provider_id: str = RECORDED_SOURCE_PROVIDER_ID
|
||||
|
||||
@@ -111,16 +159,26 @@ class RecordedRavnoves00Source:
|
||||
pacing: ReplayPacing = ReplayPacing.UNCAPPED,
|
||||
expected_frame_count: int = DEFAULT_FRAME_COUNT,
|
||||
expected_source_pack_sha256: str | None = RECORDED_SOURCE_PACK_SHA256,
|
||||
target_rate_hz: float | None = None,
|
||||
pacing_observer: SourcePacingObserver | None = None,
|
||||
clock_ns: Callable[[], int] = time.monotonic_ns,
|
||||
wait: WaitFunction | None = None,
|
||||
) -> None:
|
||||
if expected_frame_count < 1:
|
||||
raise RecordedSourceError("expected frame count must be positive")
|
||||
if target_rate_hz is not None and (
|
||||
not np.isfinite(target_rate_hz) or target_rate_hz <= 0
|
||||
):
|
||||
raise RecordedSourceError("target replay rate must be positive and finite")
|
||||
if target_rate_hz is not None and pacing is not ReplayPacing.ONE_X:
|
||||
raise RecordedSourceError("target replay rate requires paced replay")
|
||||
self.camera_index_path = camera_index_path.resolve()
|
||||
self.source_pack_path = source_pack_path.resolve()
|
||||
self.pacing = pacing
|
||||
self.expected_frame_count = expected_frame_count
|
||||
self.expected_source_pack_sha256 = expected_source_pack_sha256
|
||||
self.target_rate_hz = target_rate_hz
|
||||
self.pacing_observer = pacing_observer
|
||||
self._clock_ns = clock_ns
|
||||
self._wait = wait or _event_wait
|
||||
|
||||
@@ -130,6 +188,7 @@ class RecordedRavnoves00Source:
|
||||
repository_root: Path,
|
||||
*,
|
||||
pacing: ReplayPacing = ReplayPacing.UNCAPPED,
|
||||
target_rate_hz: float | None = None,
|
||||
) -> RecordedRavnoves00Source:
|
||||
root = repository_root.resolve()
|
||||
camera_index = (
|
||||
@@ -148,6 +207,7 @@ class RecordedRavnoves00Source:
|
||||
camera_index_path=camera_index,
|
||||
source_pack_path=source_pack,
|
||||
pacing=pacing,
|
||||
target_rate_hz=target_rate_hz,
|
||||
)
|
||||
|
||||
def packets(self, stop_event: Event) -> Iterator[SourcePacket]:
|
||||
@@ -163,6 +223,7 @@ class RecordedRavnoves00Source:
|
||||
|
||||
started_ns = int(self._clock_ns())
|
||||
source_origin_ns = _source_time_ns(timeline_rows[0])
|
||||
pacing_scale = _pacing_scale(timeline_rows, self.target_rate_hz)
|
||||
for frame_index, (camera, timeline) in enumerate(
|
||||
zip(camera_rows, timeline_rows, strict=True)
|
||||
):
|
||||
@@ -170,9 +231,20 @@ class RecordedRavnoves00Source:
|
||||
return
|
||||
packet = _packet(frame_index, camera, timeline)
|
||||
if self.pacing is ReplayPacing.ONE_X:
|
||||
target_ns = started_ns + packet.envelope.timestamps.source_ns - source_origin_ns
|
||||
source_elapsed_ns = packet.envelope.timestamps.source_ns - source_origin_ns
|
||||
target_ns = started_ns + round(source_elapsed_ns * pacing_scale)
|
||||
if not self._pace_until(stop_event, target_ns):
|
||||
return
|
||||
emitted_ns = int(self._clock_ns())
|
||||
if self.pacing_observer is not None:
|
||||
self.pacing_observer(
|
||||
SourcePacingTiming(
|
||||
sequence=packet.envelope.sequence,
|
||||
scheduled_monotonic_ns=target_ns,
|
||||
emitted_monotonic_ns=emitted_ns,
|
||||
lateness_ns=max(0, emitted_ns - target_ns),
|
||||
)
|
||||
)
|
||||
yield packet
|
||||
|
||||
def _pace_until(self, stop_event: Event, target_ns: int) -> bool:
|
||||
@@ -279,6 +351,176 @@ class PyAvRecordedImageDecoder:
|
||||
container.close()
|
||||
|
||||
|
||||
class PrefetchedRecordedImageDecoder:
|
||||
"""Decode into a bounded queue before source admission and during replay.
|
||||
|
||||
The queue absorbs ordinary storage/codec jitter without retaining the whole
|
||||
recording in RAM. Decode work remains single-pass and source order remains
|
||||
exact. The explicit phase boundary keeps cold codec initialization out of
|
||||
hot-loop latency attribution.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
decoder: RecordedImageDecoder,
|
||||
*,
|
||||
capacity_frames: int = 64,
|
||||
ready_frames: int = 64,
|
||||
timing_observer: DecodeTimingObserver | None = None,
|
||||
clock_ns: Callable[[], int] = time.perf_counter_ns,
|
||||
) -> None:
|
||||
if capacity_frames < 1:
|
||||
raise RecordedSourceError("decode prefetch capacity must be positive")
|
||||
if ready_frames < 1 or ready_frames > capacity_frames:
|
||||
raise RecordedSourceError("decode prefetch readiness must fit its capacity")
|
||||
self.decoder = decoder
|
||||
self.capacity_frames = capacity_frames
|
||||
self.ready_frames = ready_frames
|
||||
self.timing_observer = timing_observer
|
||||
self._clock_ns = clock_ns
|
||||
self._queue: Queue[NDArray[np.uint8] | _DecodeFailure | object] = Queue(
|
||||
capacity_frames
|
||||
)
|
||||
self._stop = Event()
|
||||
self._ready = Event()
|
||||
self._guard = Lock()
|
||||
self._thread: Thread | None = None
|
||||
self._phase = DecodePhase.PREADMISSION
|
||||
self._produced = 0
|
||||
self._started_ns: int | None = None
|
||||
self._ready_ns: int | None = None
|
||||
self._closed = False
|
||||
|
||||
def prepare(self, stop_event: Event) -> DecodePrefetchSnapshot:
|
||||
with self._guard:
|
||||
if self._closed:
|
||||
raise RecordedSourceError("decode prefetch is closed")
|
||||
if self._thread is None:
|
||||
self._started_ns = int(self._clock_ns())
|
||||
self._thread = Thread(
|
||||
target=self._produce,
|
||||
name="m48s-recorded-decode-prefetch",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
while not self._ready.wait(0.05):
|
||||
if stop_event.is_set():
|
||||
raise RecordedSourceError("decode prefetch preparation was cancelled")
|
||||
with self._guard:
|
||||
started_ns = self._started_ns
|
||||
ready_ns = self._ready_ns
|
||||
thread = self._thread
|
||||
produced = self._produced
|
||||
if started_ns is None or ready_ns is None or thread is None:
|
||||
raise RecordedSourceError("decode prefetch readiness is incomplete")
|
||||
first = self._peek()
|
||||
if isinstance(first, _DecodeFailure):
|
||||
raise RecordedSourceError("recorded camera prefetch failed") from first.error
|
||||
return DecodePrefetchSnapshot(
|
||||
capacity_frames=self.capacity_frames,
|
||||
ready_frames=self.ready_frames,
|
||||
buffered_frames=min(produced, self.capacity_frames),
|
||||
preparation_duration_ns=max(0, ready_ns - started_ns),
|
||||
producer_alive=thread.is_alive(),
|
||||
)
|
||||
|
||||
def mark_admission_started(self) -> None:
|
||||
with self._guard:
|
||||
if not self._ready.is_set():
|
||||
raise RecordedSourceError("source admission started before decode prefetch")
|
||||
self._phase = DecodePhase.HOT_LOOP
|
||||
|
||||
def frames(self, stop_event: Event) -> Iterator[NDArray[np.uint8]]:
|
||||
self.prepare(stop_event)
|
||||
while not stop_event.is_set() and not self._stop.is_set():
|
||||
try:
|
||||
item = self._queue.get(timeout=0.05)
|
||||
except Empty:
|
||||
continue
|
||||
try:
|
||||
if item is _DECODE_END:
|
||||
return
|
||||
if isinstance(item, _DecodeFailure):
|
||||
raise RecordedSourceError("recorded camera decode failed") from item.error
|
||||
yield cast(NDArray[np.uint8], item)
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
|
||||
def close(self, *, timeout_seconds: float = 5.0) -> None:
|
||||
with self._guard:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
self._stop.set()
|
||||
thread = self._thread
|
||||
if thread is not None:
|
||||
thread.join(timeout=max(0.0, timeout_seconds))
|
||||
if thread.is_alive():
|
||||
raise RecordedSourceError("decode prefetch worker did not stop")
|
||||
|
||||
def _produce(self) -> None:
|
||||
sequence = 0
|
||||
images = self.decoder.frames(self._stop)
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
while self._queue.full() and not self._stop.wait(0.01):
|
||||
pass
|
||||
if self._stop.is_set():
|
||||
return
|
||||
started_ns = int(self._clock_ns())
|
||||
try:
|
||||
image = next(images)
|
||||
except StopIteration:
|
||||
self._signal_ready()
|
||||
self._put(_DECODE_END)
|
||||
return
|
||||
completed_ns = int(self._clock_ns())
|
||||
with self._guard:
|
||||
phase = self._phase
|
||||
if self.timing_observer is not None:
|
||||
self.timing_observer(
|
||||
DecodedFrameTiming(
|
||||
sequence=sequence,
|
||||
duration_ns=max(0, completed_ns - started_ns),
|
||||
phase=phase,
|
||||
)
|
||||
)
|
||||
if not self._put(np.asarray(image, dtype=np.uint8)):
|
||||
return
|
||||
sequence += 1
|
||||
with self._guard:
|
||||
self._produced = sequence
|
||||
ready = sequence >= self.ready_frames
|
||||
if ready:
|
||||
self._signal_ready()
|
||||
except BaseException as exc:
|
||||
self._put(_DecodeFailure(exc))
|
||||
self._signal_ready()
|
||||
finally:
|
||||
close = getattr(images, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
|
||||
def _put(self, item: NDArray[np.uint8] | _DecodeFailure | object) -> bool:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
self._queue.put(item, timeout=0.05)
|
||||
return True
|
||||
except Full:
|
||||
continue
|
||||
return False
|
||||
|
||||
def _signal_ready(self) -> None:
|
||||
with self._guard:
|
||||
if self._ready_ns is None:
|
||||
self._ready_ns = int(self._clock_ns())
|
||||
self._ready.set()
|
||||
|
||||
def _peek(self) -> NDArray[np.uint8] | _DecodeFailure | object | None:
|
||||
with self._queue.mutex:
|
||||
return self._queue.queue[0] if self._queue.queue else None
|
||||
|
||||
|
||||
def _packet(
|
||||
frame_index: int,
|
||||
camera: dict[str, object],
|
||||
@@ -349,6 +591,21 @@ def _source_time_ns(document: _SourceTimelineRow) -> int:
|
||||
return round(value * 1_000_000_000)
|
||||
|
||||
|
||||
def _pacing_scale(
|
||||
timeline: tuple[_SourceTimelineRow, ...],
|
||||
target_rate_hz: float | None,
|
||||
) -> float:
|
||||
if target_rate_hz is None:
|
||||
return 1.0
|
||||
if len(timeline) < 2:
|
||||
raise RecordedSourceError("target-rate replay requires at least two frames")
|
||||
duration_seconds = timeline[-1].session_seconds - timeline[0].session_seconds
|
||||
if not np.isfinite(duration_seconds) or duration_seconds <= 0:
|
||||
raise RecordedSourceError("source timeline duration is invalid")
|
||||
recorded_rate_hz = (len(timeline) - 1) / duration_seconds
|
||||
return recorded_rate_hz / target_rate_hz
|
||||
|
||||
|
||||
def _integer(document: dict[str, object], key: str) -> int:
|
||||
value = document.get(key)
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
"""Fail-closed health watchdog for the canonical Mission Core service process."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import stat
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Event, Lock, Thread
|
||||
from typing import Final
|
||||
|
||||
WATCHDOG_SCHEMA: Final = "missioncore.local-service-watchdog/v1"
|
||||
WATCHDOG_ENV: Final = "MISSIONCORE_SERVICE_WATCHDOG"
|
||||
WATCHDOG_ENABLED_VALUE: Final = "1"
|
||||
MISSION_CORE_SERVICE_ID: Final = "mission-core-control-plane"
|
||||
DEFAULT_JOURNAL_MAX_BYTES: Final = 4 * 1024 * 1024
|
||||
|
||||
|
||||
class MissionCoreWatchdogError(RuntimeError):
|
||||
"""The watchdog cannot establish a trustworthy local safety boundary."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MissionCoreWatchdogPolicy:
|
||||
startup_grace_seconds: float = 45.0
|
||||
probe_interval_seconds: float = 2.0
|
||||
probe_timeout_seconds: float = 1.0
|
||||
consecutive_failure_limit: int = 3
|
||||
graceful_shutdown_seconds: float = 12.0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if (
|
||||
self.startup_grace_seconds <= 0
|
||||
or self.probe_interval_seconds <= 0
|
||||
or self.probe_timeout_seconds <= 0
|
||||
or self.consecutive_failure_limit < 1
|
||||
or self.graceful_shutdown_seconds <= 0
|
||||
):
|
||||
raise ValueError("Mission Core watchdog policy must be positive")
|
||||
|
||||
|
||||
class ConsecutiveHealthGate:
|
||||
"""Trigger only after a bounded sequence of genuine probe failures."""
|
||||
|
||||
def __init__(self, failure_limit: int) -> None:
|
||||
if failure_limit < 1:
|
||||
raise ValueError("health failure limit must be positive")
|
||||
self.failure_limit = failure_limit
|
||||
self.consecutive_failures = 0
|
||||
|
||||
def observe(self, healthy: bool) -> bool:
|
||||
if healthy:
|
||||
self.consecutive_failures = 0
|
||||
return False
|
||||
self.consecutive_failures += 1
|
||||
return self.consecutive_failures >= self.failure_limit
|
||||
|
||||
|
||||
class MissionCoreWatchdogJournal:
|
||||
"""Append bounded, private lifecycle evidence outside the application log."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: Path,
|
||||
*,
|
||||
max_bytes: int = DEFAULT_JOURNAL_MAX_BYTES,
|
||||
) -> None:
|
||||
if max_bytes < 1:
|
||||
raise ValueError("watchdog journal limit must be positive")
|
||||
self.path = path.expanduser().absolute()
|
||||
self.max_bytes = max_bytes
|
||||
self._guard = Lock()
|
||||
self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
metadata = self.path.parent.lstat()
|
||||
if not stat.S_ISDIR(metadata.st_mode):
|
||||
raise MissionCoreWatchdogError("watchdog journal parent is not a directory")
|
||||
|
||||
def append(self, event: str, **details: object) -> None:
|
||||
document = {
|
||||
"schema_version": WATCHDOG_SCHEMA,
|
||||
"event": event,
|
||||
"utc_ns": time.time_ns(),
|
||||
"monotonic_ns": time.monotonic_ns(),
|
||||
"pid": os.getpid(),
|
||||
**details,
|
||||
}
|
||||
payload = json.dumps(
|
||||
document,
|
||||
ensure_ascii=True,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
).encode("utf-8") + b"\n"
|
||||
with self._guard:
|
||||
self._rotate_if_needed(len(payload))
|
||||
flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_CLOEXEC", 0)
|
||||
flags |= getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(self.path, flags, 0o600)
|
||||
try:
|
||||
metadata = os.fstat(descriptor)
|
||||
if (
|
||||
not stat.S_ISREG(metadata.st_mode)
|
||||
or stat.S_IMODE(metadata.st_mode) != 0o600
|
||||
or metadata.st_nlink != 1
|
||||
):
|
||||
raise MissionCoreWatchdogError(
|
||||
"watchdog journal is not a private regular file"
|
||||
)
|
||||
os.write(descriptor, payload)
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
def _rotate_if_needed(self, incoming_bytes: int) -> None:
|
||||
try:
|
||||
metadata = self.path.lstat()
|
||||
except FileNotFoundError:
|
||||
return
|
||||
if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1:
|
||||
raise MissionCoreWatchdogError("watchdog journal identity changed")
|
||||
if metadata.st_size + incoming_bytes <= self.max_bytes:
|
||||
return
|
||||
previous = self.path.with_name(f"{self.path.name}.1")
|
||||
with suppress(FileNotFoundError):
|
||||
previous.unlink()
|
||||
os.replace(self.path, previous)
|
||||
|
||||
|
||||
Probe = Callable[[], bool]
|
||||
SignalAction = Callable[[], None]
|
||||
|
||||
|
||||
class MissionCoreSelfWatchdog:
|
||||
"""Terminate a live-but-unhealthy service so its init system can restart it."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repository_root: Path,
|
||||
*,
|
||||
policy: MissionCoreWatchdogPolicy | None = None,
|
||||
probe: Probe | None = None,
|
||||
request_shutdown: SignalAction | None = None,
|
||||
force_shutdown: SignalAction | None = None,
|
||||
journal: MissionCoreWatchdogJournal | None = None,
|
||||
) -> None:
|
||||
self.policy = policy or MissionCoreWatchdogPolicy()
|
||||
self.probe = probe or _mission_core_health_probe(
|
||||
self.policy.probe_timeout_seconds
|
||||
)
|
||||
self.request_shutdown = request_shutdown or _process_group_signal(signal.SIGTERM)
|
||||
self.force_shutdown = force_shutdown or _process_group_signal(signal.SIGKILL)
|
||||
self.journal = journal or MissionCoreWatchdogJournal(
|
||||
repository_root.expanduser().absolute()
|
||||
/ ".runtime/mission-core/service-watchdog.jsonl"
|
||||
)
|
||||
self._stop = Event()
|
||||
self._thread = Thread(
|
||||
target=self._run,
|
||||
name="mission-core-self-health-watchdog",
|
||||
daemon=True,
|
||||
)
|
||||
self._started = False
|
||||
|
||||
def start(self) -> None:
|
||||
if self._started:
|
||||
raise MissionCoreWatchdogError("Mission Core watchdog already started")
|
||||
self._started = True
|
||||
self.journal.append("watchdog-started", policy=_policy_dict(self.policy))
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
if not self._started:
|
||||
return
|
||||
self._stop.set()
|
||||
self._thread.join(timeout=self.policy.probe_timeout_seconds + 1.0)
|
||||
self.journal.append(
|
||||
"watchdog-stopped",
|
||||
worker_alive=self._thread.is_alive(),
|
||||
)
|
||||
|
||||
def _run(self) -> None:
|
||||
if self._stop.wait(self.policy.startup_grace_seconds):
|
||||
return
|
||||
gate = ConsecutiveHealthGate(self.policy.consecutive_failure_limit)
|
||||
last_reported_health: bool | None = None
|
||||
while not self._stop.is_set():
|
||||
healthy = False
|
||||
try:
|
||||
healthy = self.probe()
|
||||
except Exception:
|
||||
healthy = False
|
||||
triggered = gate.observe(healthy)
|
||||
if healthy != last_reported_health:
|
||||
self.journal.append(
|
||||
"health-state-changed",
|
||||
healthy=healthy,
|
||||
consecutive_failures=gate.consecutive_failures,
|
||||
)
|
||||
last_reported_health = healthy
|
||||
if triggered:
|
||||
self.journal.append(
|
||||
"restart-requested",
|
||||
reason="consecutive-health-probe-failures",
|
||||
consecutive_failures=gate.consecutive_failures,
|
||||
)
|
||||
self.request_shutdown()
|
||||
if not self._stop.wait(self.policy.graceful_shutdown_seconds):
|
||||
self.journal.append(
|
||||
"restart-escalated",
|
||||
reason="graceful-shutdown-timeout",
|
||||
)
|
||||
self.force_shutdown()
|
||||
return
|
||||
if self._stop.wait(self.policy.probe_interval_seconds):
|
||||
return
|
||||
|
||||
|
||||
def watchdog_enabled(environ: Mapping[str, str] = os.environ) -> bool:
|
||||
return environ.get(WATCHDOG_ENV) == WATCHDOG_ENABLED_VALUE
|
||||
|
||||
|
||||
def _mission_core_health_probe(timeout_seconds: float) -> Probe:
|
||||
def probe() -> bool:
|
||||
connection = http.client.HTTPConnection(
|
||||
"127.0.0.1",
|
||||
8000,
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
try:
|
||||
connection.request("GET", "/api/health", headers={"Connection": "close"})
|
||||
response = connection.getresponse()
|
||||
payload = response.read(64 * 1024 + 1)
|
||||
except (OSError, TimeoutError, http.client.HTTPException):
|
||||
return False
|
||||
finally:
|
||||
connection.close()
|
||||
if response.status != 200 or len(payload) > 64 * 1024:
|
||||
return False
|
||||
try:
|
||||
document = json.loads(payload)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return False
|
||||
return bool(
|
||||
isinstance(document, dict)
|
||||
and document.get("ok") is True
|
||||
and document.get("status") == "ok"
|
||||
and document.get("service") == MISSION_CORE_SERVICE_ID
|
||||
)
|
||||
|
||||
return probe
|
||||
|
||||
|
||||
def _process_group_signal(signal_number: signal.Signals) -> SignalAction:
|
||||
def send() -> None:
|
||||
os.killpg(os.getpgrp(), signal_number)
|
||||
|
||||
return send
|
||||
|
||||
|
||||
def _policy_dict(policy: MissionCoreWatchdogPolicy) -> dict[str, object]:
|
||||
return {
|
||||
"startup_grace_seconds": policy.startup_grace_seconds,
|
||||
"probe_interval_seconds": policy.probe_interval_seconds,
|
||||
"probe_timeout_seconds": policy.probe_timeout_seconds,
|
||||
"consecutive_failure_limit": policy.consecutive_failure_limit,
|
||||
"graceful_shutdown_seconds": policy.graceful_shutdown_seconds,
|
||||
}
|
||||
@@ -305,6 +305,7 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
|
||||
decision = manifest.get("decision")
|
||||
method = manifest.get("method")
|
||||
metrics = manifest.get("metrics")
|
||||
load_envelope = metrics.get("load_envelope") if isinstance(metrics, dict) else None
|
||||
status = manifest.get("status")
|
||||
integrated = status == INTEGRATED_STATUS
|
||||
if (
|
||||
@@ -331,12 +332,35 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
|
||||
or decision.get("integrated_world_state_gate_evaluated") is not integrated
|
||||
or (integrated and decision.get("integrated_world_state_gate_passed") is not True)
|
||||
or (integrated and decision.get("detector_replacement_authorized") is not False)
|
||||
or (
|
||||
load_envelope is not None
|
||||
and (
|
||||
decision.get("load_envelope_evaluated") is not True
|
||||
or not isinstance(load_envelope, dict)
|
||||
or decision.get("production_rate_repeatability_passed")
|
||||
is not load_envelope.get("production_rate_repeatability_passed")
|
||||
or decision.get("reserve_12_fps_passed")
|
||||
is not load_envelope.get("reserve_12_fps_passed")
|
||||
or decision.get("limit_15_fps_passed")
|
||||
is not load_envelope.get("limit_15_fps_passed")
|
||||
or decision.get("load_envelope_accepted")
|
||||
is not load_envelope.get("load_envelope_accepted")
|
||||
)
|
||||
)
|
||||
or decision.get("production_accepted") is not False
|
||||
or not isinstance(method, dict)
|
||||
or method.get("schema_version") != "missioncore.laboratory-method/v1"
|
||||
or method.get("completeness") != "complete"
|
||||
or not isinstance(metrics, dict)
|
||||
or (integrated and not isinstance(metrics.get("integrated_world_state"), dict))
|
||||
or (
|
||||
metrics.get("runtime_hardening") is not None
|
||||
and not _valid_runtime_hardening(metrics["runtime_hardening"])
|
||||
)
|
||||
or (
|
||||
metrics.get("load_envelope") is not None
|
||||
and not _valid_load_envelope(metrics["load_envelope"])
|
||||
)
|
||||
or not isinstance(manifest.get("limitations"), list)
|
||||
or not isinstance(catalog_descriptor, dict)
|
||||
or catalog_descriptor.get("path") != "catalog.json"
|
||||
@@ -363,6 +387,217 @@ def _load_result_uncached(candidate: Path) -> dict[str, Any]:
|
||||
return {"manifest": manifest, "catalog": catalog}
|
||||
|
||||
|
||||
def _valid_runtime_hardening(value: object) -> bool:
|
||||
if not isinstance(value, dict) or value.get("schema_version") != (
|
||||
"missioncore.m48s-runtime-hardening-comparison/v1"
|
||||
):
|
||||
return False
|
||||
full_run_keys = {
|
||||
"source_frames_admitted",
|
||||
"delivered_world_states",
|
||||
"superseded_frames",
|
||||
"effective_world_state_fps",
|
||||
"world_state_completion_age_p95_ms",
|
||||
"world_state_completion_age_p99_ms",
|
||||
"world_state_completion_age_maximum_ms",
|
||||
"rolling_maximum_ms",
|
||||
"geometry_maximum_ms",
|
||||
"additional_inference_passes",
|
||||
}
|
||||
for name in ("baseline", "hardened"):
|
||||
run = value.get(name)
|
||||
if (
|
||||
not isinstance(run, dict)
|
||||
or set(run) != full_run_keys
|
||||
or any(not _nonnegative_number(item) for item in run.values())
|
||||
):
|
||||
return False
|
||||
startup = value.get("startup")
|
||||
if not isinstance(startup, dict) or set(startup) != {
|
||||
"baseline",
|
||||
"prewarmed",
|
||||
"prewarm_duration_ms",
|
||||
"prewarm_inference_passes",
|
||||
"validation_frames",
|
||||
}:
|
||||
return False
|
||||
for name in ("baseline", "prewarmed"):
|
||||
frame = startup.get(name)
|
||||
if (
|
||||
not isinstance(frame, dict)
|
||||
or set(frame) != {"detector_ms", "world_state_ms"}
|
||||
or any(not _nonnegative_number(item) for item in frame.values())
|
||||
):
|
||||
return False
|
||||
return (
|
||||
_nonnegative_number(startup.get("prewarm_duration_ms"))
|
||||
and isinstance(startup.get("prewarm_inference_passes"), int)
|
||||
and not isinstance(startup.get("prewarm_inference_passes"), bool)
|
||||
and startup["prewarm_inference_passes"] > 0
|
||||
and isinstance(startup.get("validation_frames"), int)
|
||||
and not isinstance(startup.get("validation_frames"), bool)
|
||||
and startup["validation_frames"] > 0
|
||||
)
|
||||
|
||||
|
||||
def _valid_load_envelope(value: object) -> bool:
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
if value.get("schema_version") == "missioncore.m48s-load-envelope-comparison/v1":
|
||||
return _valid_legacy_load_envelope(value)
|
||||
if value.get("schema_version") != "missioncore.m48s-load-envelope-comparison/v2":
|
||||
return False
|
||||
if (
|
||||
value.get("repeat_count_per_rate") != 3
|
||||
or value.get("production_rate_repeatability_passed") is not True
|
||||
or value.get("reserve_12_fps_passed") is not True
|
||||
or value.get("limit_15_fps_passed") is not True
|
||||
or value.get("load_envelope_accepted") is not True
|
||||
or value.get("compute_capacity_at_least_fps") != 15.0
|
||||
or value.get("bottleneck_interpretation")
|
||||
!= "cold-video-decode-isolated-before-admission-no-steady-gpu-saturation"
|
||||
):
|
||||
return False
|
||||
scenarios = value.get("scenarios")
|
||||
scenario_ids = [item.get("id") for item in scenarios if isinstance(item, dict)] if (
|
||||
isinstance(scenarios, list)
|
||||
) else []
|
||||
if scenario_ids != ["production-10fps", "reserve-12fps", "limit-15fps"]:
|
||||
return False
|
||||
required_numeric = {
|
||||
"requested_source_rate_hz",
|
||||
"source_frames_admitted",
|
||||
"delivered_world_states",
|
||||
"superseded_frames",
|
||||
"delivery_ratio",
|
||||
"effective_world_state_fps",
|
||||
"world_state_completion_age_p95_ms",
|
||||
"world_state_completion_age_p99_ms",
|
||||
"world_state_completion_age_maximum_ms",
|
||||
"hot_loop_decode_p95_ms",
|
||||
"hot_loop_decode_maximum_ms",
|
||||
"preadmission_decode_maximum_ms",
|
||||
"source_prefetch_preparation_maximum_ms",
|
||||
"source_pacing_lateness_p95_ms",
|
||||
"source_pacing_lateness_maximum_ms",
|
||||
"detector_p95_ms",
|
||||
"detector_maximum_ms",
|
||||
"gpu_utilization_mean_percent",
|
||||
"gpu_utilization_maximum_percent",
|
||||
"gpu_memory_maximum_mib",
|
||||
"process_peak_rss_mib",
|
||||
"queue_capacity",
|
||||
"additional_inference_passes",
|
||||
}
|
||||
for scenario in scenarios:
|
||||
repetitions = scenario.get("repetitions") if isinstance(scenario, dict) else None
|
||||
if (
|
||||
not isinstance(scenario, dict)
|
||||
or scenario.get("repeat_count") != 3
|
||||
or scenario.get("all_repetitions_passed") is not True
|
||||
or scenario.get("integrity_gate_passed") is not True
|
||||
or scenario.get("operating_target_gate_passed") is not True
|
||||
or scenario.get("additional_inference_passes") != 0
|
||||
or scenario.get("queue_capacity") != 2
|
||||
or not isinstance(scenario.get("queue_high_watermarks"), dict)
|
||||
or any(value != 2 for value in scenario["queue_high_watermarks"].values())
|
||||
or not isinstance(scenario.get("thresholds"), dict)
|
||||
or any(not _nonnegative_number(scenario.get(key)) for key in required_numeric)
|
||||
or not isinstance(repetitions, list)
|
||||
or len(repetitions) != 3
|
||||
):
|
||||
return False
|
||||
for expected_repetition, repetition in enumerate(repetitions, start=1):
|
||||
if (
|
||||
not isinstance(repetition, dict)
|
||||
or repetition.get("repetition") != expected_repetition
|
||||
or repetition.get("operating_target_gate_passed") is not True
|
||||
or not isinstance(repetition.get("run_id"), str)
|
||||
or not repetition["run_id"]
|
||||
or any(
|
||||
not _nonnegative_number(repetition.get(key))
|
||||
for key in {
|
||||
"delivered_world_states",
|
||||
"superseded_frames",
|
||||
"delivery_ratio",
|
||||
"effective_world_state_fps",
|
||||
"world_state_completion_age_p95_ms",
|
||||
"hot_loop_decode_p95_ms",
|
||||
"hot_loop_decode_maximum_ms",
|
||||
"preadmission_decode_maximum_ms",
|
||||
"source_pacing_lateness_maximum_ms",
|
||||
}
|
||||
)
|
||||
or any(
|
||||
not isinstance(repetition.get(key), str)
|
||||
or SHA256.fullmatch(repetition[key]) is None
|
||||
for key in ("result_sha256", "frame_evidence_sha256")
|
||||
)
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _valid_legacy_load_envelope(value: dict[str, object]) -> bool:
|
||||
if (
|
||||
value.get("production_rate_repeatability_passed") is not False
|
||||
or value.get("reserve_12_fps_passed") is not True
|
||||
or value.get("limit_15_fps_passed") is not True
|
||||
or value.get("load_envelope_accepted") is not False
|
||||
or value.get("compute_capacity_at_least_fps") != 15.0
|
||||
or value.get("bottleneck_interpretation")
|
||||
!= "rare-source-decode-or-scheduling-tail-not-steady-gpu-saturation"
|
||||
):
|
||||
return False
|
||||
scenarios = value.get("scenarios")
|
||||
if not isinstance(scenarios, list) or [
|
||||
item.get("id") for item in scenarios if isinstance(item, dict)
|
||||
] != ["production-10fps", "reserve-12fps", "limit-15fps"]:
|
||||
return False
|
||||
expected_targets = (False, True, True)
|
||||
required_numeric = {
|
||||
"requested_source_rate_hz",
|
||||
"source_frames_admitted",
|
||||
"delivered_world_states",
|
||||
"superseded_frames",
|
||||
"delivery_ratio",
|
||||
"effective_world_state_fps",
|
||||
"world_state_completion_age_p95_ms",
|
||||
"world_state_completion_age_p99_ms",
|
||||
"world_state_completion_age_maximum_ms",
|
||||
"decode_p95_ms",
|
||||
"decode_maximum_ms",
|
||||
"detector_p95_ms",
|
||||
"detector_maximum_ms",
|
||||
"gpu_utilization_mean_percent",
|
||||
"gpu_utilization_maximum_percent",
|
||||
"gpu_memory_maximum_mib",
|
||||
"process_peak_rss_mib",
|
||||
"queue_capacity",
|
||||
"additional_inference_passes",
|
||||
}
|
||||
for scenario, target in zip(scenarios, expected_targets, strict=True):
|
||||
if (
|
||||
not isinstance(scenario, dict)
|
||||
or scenario.get("integrity_gate_passed") is not True
|
||||
or scenario.get("operating_target_gate_passed") is not target
|
||||
or scenario.get("additional_inference_passes") != 0
|
||||
or scenario.get("queue_capacity") != 2
|
||||
or not isinstance(scenario.get("queue_high_watermarks"), dict)
|
||||
or any(depth != 2 for depth in scenario["queue_high_watermarks"].values())
|
||||
or not isinstance(scenario.get("thresholds"), dict)
|
||||
or any(not _nonnegative_number(scenario.get(key)) for key in required_numeric)
|
||||
or not isinstance(scenario.get("frame_evidence_sha256"), str)
|
||||
or SHA256.fullmatch(scenario["frame_evidence_sha256"]) is None
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _nonnegative_number(value: object) -> bool:
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool) and float(value) >= 0.0
|
||||
|
||||
|
||||
def _candidate_signature(candidate: Path) -> tuple[int, ...]:
|
||||
if not candidate.is_dir() or candidate.is_symlink():
|
||||
raise RuntimeError("M4.8S result candidate is invalid")
|
||||
|
||||
@@ -83,6 +83,7 @@ def test_serve_resolves_frontend_from_repository_root(monkeypatch: Any) -> None:
|
||||
"port": 8000,
|
||||
"log_level": "info",
|
||||
"access_log": True,
|
||||
"timeout_graceful_shutdown": 10,
|
||||
}
|
||||
assert lease.active is False
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import plistlib
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.local_service_launchd import plan_mission_core_launch_agent
|
||||
|
||||
|
||||
def test_launch_agent_plan_disables_sync_and_enables_watchdog(tmp_path: Path) -> None:
|
||||
repository = tmp_path / "repo"
|
||||
repository.mkdir()
|
||||
uv_entrypoint = tmp_path / "uv"
|
||||
uv_entrypoint.write_text("#!/bin/sh\n")
|
||||
uv_entrypoint.chmod(0o700)
|
||||
agent = tmp_path / "agent.plist"
|
||||
agent.write_bytes(
|
||||
plistlib.dumps(
|
||||
{
|
||||
"Label": "com.nodedc.mission-core.local",
|
||||
"ProgramArguments": [str(uv_entrypoint), "run", "k1link", "serve"],
|
||||
"WorkingDirectory": str(repository),
|
||||
"EnvironmentVariables": {"PATH": "/usr/bin:/bin"},
|
||||
}
|
||||
)
|
||||
)
|
||||
agent.chmod(0o600)
|
||||
|
||||
plan = plan_mission_core_launch_agent(
|
||||
repository_root=repository,
|
||||
agent_path=agent,
|
||||
)
|
||||
desired = plistlib.loads(plan.desired_payload)
|
||||
|
||||
assert desired["ProgramArguments"] == [
|
||||
str(uv_entrypoint),
|
||||
"run",
|
||||
"--no-sync",
|
||||
"k1link",
|
||||
"serve",
|
||||
]
|
||||
assert desired["EnvironmentVariables"]["MISSIONCORE_SERVICE_WATCHDOG"] == "1"
|
||||
assert desired["KeepAlive"] is True
|
||||
assert desired["RunAtLoad"] is True
|
||||
assert desired["AbandonProcessGroup"] is False
|
||||
assert desired["ExitTimeOut"] == 20
|
||||
assert plan.current_sha256 != plan.desired_sha256
|
||||
@@ -38,7 +38,7 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N
|
||||
assert manifest["result_id"] == result.result_id
|
||||
assert result.result_id.endswith(identity_digest)
|
||||
assert manifest["identity_sha256"] == identity_digest
|
||||
assert len(manifest["artifacts"]) == 31
|
||||
assert len(manifest["artifacts"]) == 45
|
||||
assert manifest["method"]["completeness"] == "complete"
|
||||
assert manifest["bounded_question_accepted"] is True
|
||||
assert manifest["ground_truth"] is False
|
||||
@@ -49,6 +49,11 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N
|
||||
"integrated_world_state_gate_evaluated": True,
|
||||
"integrated_world_state_gate_passed": True,
|
||||
"full_replay_visual_published": True,
|
||||
"load_envelope_evaluated": True,
|
||||
"production_rate_repeatability_passed": True,
|
||||
"reserve_12_fps_passed": True,
|
||||
"limit_15_fps_passed": True,
|
||||
"load_envelope_accepted": True,
|
||||
"detector_replacement_authorized": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
@@ -66,6 +71,47 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N
|
||||
assert max(integrated["queue_high_watermarks"].values()) <= 2
|
||||
assert integrated["additional_inference_passes"] == 0
|
||||
assert integrated["failures"] == 0
|
||||
hardening = manifest["metrics"]["runtime_hardening"]
|
||||
assert hardening["schema_version"] == "missioncore.m48s-runtime-hardening-comparison/v1"
|
||||
assert hardening["baseline"]["delivered_world_states"] == 4_480
|
||||
assert hardening["baseline"]["superseded_frames"] == 9
|
||||
assert hardening["hardened"]["delivered_world_states"] == 4_488
|
||||
assert hardening["hardened"]["superseded_frames"] == 1
|
||||
assert hardening["baseline"]["rolling_maximum_ms"] == 762.638263
|
||||
assert hardening["hardened"]["rolling_maximum_ms"] == 28.044958
|
||||
assert hardening["startup"]["baseline"]["detector_ms"] == 397.356888
|
||||
assert hardening["startup"]["prewarmed"]["detector_ms"] == 27.662887
|
||||
assert hardening["startup"]["baseline"]["world_state_ms"] == 449.893287
|
||||
assert hardening["startup"]["prewarmed"]["world_state_ms"] == 60.516957
|
||||
assert hardening["startup"]["prewarm_duration_ms"] == 420.721918
|
||||
assert hardening["startup"]["validation_frames"] == 1_000
|
||||
envelope = manifest["metrics"]["load_envelope"]
|
||||
assert envelope["schema_version"] == "missioncore.m48s-load-envelope-comparison/v2"
|
||||
assert envelope["repeat_count_per_rate"] == 3
|
||||
assert envelope["production_rate_repeatability_passed"] is True
|
||||
assert envelope["reserve_12_fps_passed"] is True
|
||||
assert envelope["limit_15_fps_passed"] is True
|
||||
assert envelope["load_envelope_accepted"] is True
|
||||
assert envelope["compute_capacity_at_least_fps"] == 15.0
|
||||
production, reserve, limit = envelope["scenarios"]
|
||||
assert production["delivered_world_states"] >= 4_485
|
||||
assert production["superseded_frames"] <= 5
|
||||
assert production["operating_target_gate_passed"] is True
|
||||
assert reserve["delivered_world_states"] >= 4_467
|
||||
assert reserve["superseded_frames"] <= 22
|
||||
assert reserve["operating_target_gate_passed"] is True
|
||||
assert limit["delivered_world_states"] == 4_488
|
||||
assert limit["superseded_frames"] == 1
|
||||
assert limit["operating_target_gate_passed"] is True
|
||||
assert all(item["repeat_count"] == 3 for item in envelope["scenarios"])
|
||||
assert all(len(item["repetitions"]) == 3 for item in envelope["scenarios"])
|
||||
assert all(item["all_repetitions_passed"] is True for item in envelope["scenarios"])
|
||||
assert all(item["hot_loop_decode_p95_ms"] < 30 for item in envelope["scenarios"])
|
||||
assert all(
|
||||
item["source_prefetch_preparation_maximum_ms"] > 3_000
|
||||
for item in envelope["scenarios"]
|
||||
)
|
||||
assert all(item["additional_inference_passes"] == 0 for item in envelope["scenarios"])
|
||||
|
||||
catalog = json.loads((result.result_root / "catalog.json").read_text("utf-8"))
|
||||
assert catalog["frame_count"] == len(FRAME_IDS) == 11
|
||||
@@ -85,7 +131,7 @@ def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> N
|
||||
)
|
||||
proof = verify_laboratory_evidence_result(definition, result.result_root)
|
||||
assert proof["result_id"] == result.result_id
|
||||
assert proof["artifact_count"] == 31
|
||||
assert proof["artifact_count"] == 45
|
||||
|
||||
with pytest.raises(M48SFixedClassDetectorLabError, match="already exists"):
|
||||
build_m48s_fixed_class_detector_lab(
|
||||
|
||||
@@ -47,20 +47,30 @@ def test_m48s_lab_api_projects_verified_result_frame_and_camera(tmp_path: Path)
|
||||
result.json()["metrics"]["integrated_world_state"]["world_state_completion_age_p95_ms"]
|
||||
== 74.733648
|
||||
)
|
||||
hardening = result.json()["metrics"]["runtime_hardening"]
|
||||
assert hardening["baseline"]["delivered_world_states"] == 4_480
|
||||
assert hardening["hardened"]["delivered_world_states"] == 4_488
|
||||
assert hardening["startup"]["prewarmed"]["world_state_ms"] == 60.516957
|
||||
envelope = result.json()["metrics"]["load_envelope"]
|
||||
assert envelope["production_rate_repeatability_passed"] is True
|
||||
assert envelope["reserve_12_fps_passed"] is True
|
||||
assert envelope["limit_15_fps_passed"] is True
|
||||
assert envelope["load_envelope_accepted"] is True
|
||||
assert envelope["repeat_count_per_rate"] == 3
|
||||
assert [item["requested_source_rate_hz"] for item in envelope["scenarios"]] == [
|
||||
10.0,
|
||||
12.0,
|
||||
15.0,
|
||||
]
|
||||
assert result.json()["ground_truth"] is False
|
||||
assert len(result.json()["frames"]) == 11
|
||||
|
||||
timeline = client.get(
|
||||
f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/timeline"
|
||||
)
|
||||
timeline = client.get(f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/timeline")
|
||||
assert timeline.status_code == 200
|
||||
assert timeline.json()["frame_count"] == 4_489
|
||||
assert timeline.json()["world_state_frame_count"] == 4_481
|
||||
assert timeline.json()["superseded_frame_count"] == 8
|
||||
assert (
|
||||
timeline.json()["camera_point_delivery"]
|
||||
== "factory-kb4-causal-registered-accumulation"
|
||||
)
|
||||
assert timeline.json()["camera_point_delivery"] == "factory-kb4-causal-registered-accumulation"
|
||||
assert timeline.json()["camera_point_window_seconds"] == 2.0
|
||||
assert timeline.json()["camera_point_sample_limit"] == 20_000
|
||||
chunk = client.get(
|
||||
@@ -72,9 +82,7 @@ def test_m48s_lab_api_projects_verified_result_frame_and_camera(tmp_path: Path)
|
||||
assert replay_frame["world_state_available"] is True
|
||||
assert replay_frame["camera_projection"] == "factory-kb4-exact"
|
||||
assert replay_frame["camera_projected_sample_count"] > 0
|
||||
assert any(
|
||||
item["semantic_hint"] == "dog" for item in replay_frame["camera_proposals"]
|
||||
)
|
||||
assert any(item["semantic_hint"] == "dog" for item in replay_frame["camera_proposals"])
|
||||
camera_points = client.get(
|
||||
f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}"
|
||||
"/timeline/frames/253/camera-points"
|
||||
@@ -82,9 +90,7 @@ def test_m48s_lab_api_projects_verified_result_frame_and_camera(tmp_path: Path)
|
||||
assert camera_points.status_code == 200
|
||||
camera_point_payload = camera_points.json()
|
||||
assert camera_point_payload["schema_version"] == "missioncore.m48s-camera-point-overlay/v1"
|
||||
assert camera_point_payload["projection"] == (
|
||||
"factory-kb4-causal-registered-accumulation"
|
||||
)
|
||||
assert camera_point_payload["projection"] == ("factory-kb4-causal-registered-accumulation")
|
||||
assert camera_point_payload["source_frame_count"] > 1
|
||||
assert camera_point_payload["sample_count"] > replay_frame["camera_projected_sample_count"]
|
||||
assert camera_point_payload["sample_count"] <= 20_000
|
||||
|
||||
@@ -6,7 +6,7 @@ from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from k1link.perception.detector import DetectorFrameTiming
|
||||
from k1link.perception.recorded_source import DecodedFrameTiming
|
||||
from k1link.perception.recorded_source import DecodedFrameTiming, SourcePacingTiming
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
RUNNER_PATH = REPOSITORY_ROOT / "experiments/perception/run_m48s_reference_graph_shadow_worker.py"
|
||||
@@ -19,6 +19,14 @@ SPEC.loader.exec_module(RUNNER)
|
||||
def test_frame_timing_store_closes_provider_and_unattributed_time() -> None:
|
||||
store = RUNNER.FrameTimingStore()
|
||||
store.observe_decode(DecodedFrameTiming(sequence=7, duration_ns=5))
|
||||
store.observe_pacing(
|
||||
SourcePacingTiming(
|
||||
sequence=7,
|
||||
scheduled_monotonic_ns=80,
|
||||
emitted_monotonic_ns=90,
|
||||
lateness_ns=10,
|
||||
)
|
||||
)
|
||||
store.observe_detector(
|
||||
DetectorFrameTiming(
|
||||
sequence=7,
|
||||
@@ -50,6 +58,14 @@ def test_frame_timing_store_closes_provider_and_unattributed_time() -> None:
|
||||
def test_pipeline_timing_metrics_preserve_single_pass_stage_breakdown() -> None:
|
||||
store = RUNNER.FrameTimingStore()
|
||||
store.observe_decode(DecodedFrameTiming(sequence=3, duration_ns=1_000_000))
|
||||
store.observe_pacing(
|
||||
SourcePacingTiming(
|
||||
sequence=3,
|
||||
scheduled_monotonic_ns=0,
|
||||
emitted_monotonic_ns=1_000_000,
|
||||
lateness_ns=1_000_000,
|
||||
)
|
||||
)
|
||||
store.observe_detector(
|
||||
DetectorFrameTiming(
|
||||
sequence=3,
|
||||
@@ -79,6 +95,7 @@ def test_pipeline_timing_metrics_preserve_single_pass_stage_breakdown() -> None:
|
||||
assert metrics["detector_ms"]["inference_transport"]["maximum"] == 3.0
|
||||
assert metrics["provider_ms"]["threat"]["maximum"] == 5.0
|
||||
assert metrics["graph_unattributed_ms"]["maximum"] == 6.0
|
||||
assert metrics["delivered_source_pacing_lateness_ms"]["maximum"] == 1.0
|
||||
assert metrics["maximum_graph_sequence"] == 3
|
||||
assert metrics["additional_inference_passes"] == 0
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
_SCRIPT = Path(__file__).parents[1] / "scripts/manage_mission_core_launch_agent.py"
|
||||
_SPEC = importlib.util.spec_from_file_location(
|
||||
"manage_mission_core_launch_agent",
|
||||
_SCRIPT,
|
||||
)
|
||||
assert _SPEC is not None and _SPEC.loader is not None
|
||||
manager = importlib.util.module_from_spec(_SPEC)
|
||||
_SPEC.loader.exec_module(manager)
|
||||
|
||||
|
||||
def test_reload_waits_for_launchd_transition_before_bootstrap(
|
||||
monkeypatch: Any,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
commands: list[tuple[str, ...]] = []
|
||||
print_results = iter((0, 0, 1))
|
||||
|
||||
def fake_run(arguments: list[str], **_: object) -> subprocess.CompletedProcess[str]:
|
||||
command = tuple(arguments)
|
||||
commands.append(command)
|
||||
if arguments[1] == "print":
|
||||
return subprocess.CompletedProcess(arguments, next(print_results), "", "")
|
||||
return subprocess.CompletedProcess(arguments, 0, "", "")
|
||||
|
||||
monkeypatch.setattr(manager.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(manager.time, "sleep", lambda _: None)
|
||||
|
||||
agent = tmp_path / "agent.plist"
|
||||
manager._reload_launch_agent(agent)
|
||||
|
||||
assert [command[1] for command in commands] == [
|
||||
"bootout",
|
||||
"print",
|
||||
"print",
|
||||
"print",
|
||||
"bootstrap",
|
||||
]
|
||||
|
||||
|
||||
def test_reload_rejects_failed_bootout_while_job_is_still_loaded(
|
||||
monkeypatch: Any,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
def fake_run(arguments: list[str], **_: object) -> subprocess.CompletedProcess[str]:
|
||||
if arguments[1] == "bootout":
|
||||
return subprocess.CompletedProcess(arguments, 5, "", "")
|
||||
return subprocess.CompletedProcess(arguments, 0, "", "")
|
||||
|
||||
monkeypatch.setattr(manager.subprocess, "run", fake_run)
|
||||
|
||||
try:
|
||||
manager._reload_launch_agent(tmp_path / "agent.plist")
|
||||
except manager.MissionCoreLaunchAgentError as exc:
|
||||
assert "bootout failed with exit code 5" in str(exc)
|
||||
else:
|
||||
raise AssertionError("failed bootout was accepted")
|
||||
@@ -62,9 +62,12 @@ from k1link.perception.providers import (
|
||||
from k1link.perception.recorded_source import (
|
||||
DecodedFrameTiming,
|
||||
DecodedRecordedSource,
|
||||
DecodePhase,
|
||||
PrefetchedRecordedImageDecoder,
|
||||
RecordedRavnoves00Source,
|
||||
RecordedSourceError,
|
||||
ReplayPacing,
|
||||
SourcePacingTiming,
|
||||
)
|
||||
|
||||
|
||||
@@ -835,6 +838,92 @@ def test_recorded_source_reuses_one_timeline_for_1x_and_uncapped(tmp_path: Path)
|
||||
assert one_x[1].registered_point_increment_payload is None
|
||||
|
||||
|
||||
def test_recorded_source_target_rate_changes_only_wall_clock_pacing(tmp_path: Path) -> None:
|
||||
camera_path, timeline_path = _write_recorded_fixture(tmp_path)
|
||||
now = [1_000_000_000]
|
||||
waits: list[float] = []
|
||||
pacing: list[SourcePacingTiming] = []
|
||||
|
||||
def wait(stop_event: Event, seconds: float) -> bool:
|
||||
waits.append(seconds)
|
||||
now[0] += round(seconds * 1_000_000_000)
|
||||
return stop_event.is_set()
|
||||
|
||||
source = RecordedRavnoves00Source(
|
||||
camera_index_path=camera_path,
|
||||
source_pack_path=timeline_path,
|
||||
pacing=ReplayPacing.ONE_X,
|
||||
target_rate_hz=20.0,
|
||||
expected_frame_count=2,
|
||||
expected_source_pack_sha256=None,
|
||||
pacing_observer=pacing.append,
|
||||
clock_ns=lambda: now[0],
|
||||
wait=wait,
|
||||
)
|
||||
|
||||
packets = list(source.packets(Event()))
|
||||
|
||||
assert waits == pytest.approx([0.05])
|
||||
assert [sample.sequence for sample in pacing] == [0, 1]
|
||||
assert [sample.lateness_ns for sample in pacing] == [0, 0]
|
||||
assert (
|
||||
packets[1].envelope.timestamps.source_ns
|
||||
- packets[0].envelope.timestamps.source_ns
|
||||
== 100_000_000
|
||||
)
|
||||
|
||||
|
||||
def test_recorded_source_rejects_target_rate_for_uncapped_replay(tmp_path: Path) -> None:
|
||||
camera_path, timeline_path = _write_recorded_fixture(tmp_path)
|
||||
with pytest.raises(RecordedSourceError, match="requires paced replay"):
|
||||
RecordedRavnoves00Source(
|
||||
camera_index_path=camera_path,
|
||||
source_pack_path=timeline_path,
|
||||
pacing=ReplayPacing.UNCAPPED,
|
||||
target_rate_hz=12.0,
|
||||
expected_frame_count=2,
|
||||
expected_source_pack_sha256=None,
|
||||
)
|
||||
|
||||
|
||||
def test_prefetched_decoder_moves_cold_decode_before_source_admission() -> None:
|
||||
observed: list[DecodedFrameTiming] = []
|
||||
|
||||
class Decoder:
|
||||
def frames(self, stop_event: Event) -> Iterator[np.ndarray]:
|
||||
for index in range(4):
|
||||
if stop_event.is_set():
|
||||
return
|
||||
yield np.full((2, 3, 3), index, dtype=np.uint8)
|
||||
|
||||
decoder = PrefetchedRecordedImageDecoder(
|
||||
Decoder(),
|
||||
capacity_frames=2,
|
||||
ready_frames=2,
|
||||
timing_observer=observed.append,
|
||||
)
|
||||
stop_event = Event()
|
||||
|
||||
snapshot = decoder.prepare(stop_event)
|
||||
|
||||
assert snapshot.buffered_frames == 2
|
||||
assert snapshot.capacity_frames == 2
|
||||
assert [sample.phase for sample in observed] == [
|
||||
DecodePhase.PREADMISSION,
|
||||
DecodePhase.PREADMISSION,
|
||||
]
|
||||
decoder.mark_admission_started()
|
||||
frames = list(decoder.frames(stop_event))
|
||||
decoder.close()
|
||||
|
||||
assert [int(frame[0, 0, 0]) for frame in frames] == [0, 1, 2, 3]
|
||||
assert [sample.sequence for sample in observed] == [0, 1, 2, 3]
|
||||
assert [sample.phase for sample in observed[2:]] == [
|
||||
DecodePhase.HOT_LOOP,
|
||||
DecodePhase.HOT_LOOP,
|
||||
]
|
||||
|
||||
|
||||
def test_recorded_source_rejects_timeline_mismatch(tmp_path: Path) -> None:
|
||||
camera_path, timeline_path = _write_recorded_fixture(tmp_path, mismatched=True)
|
||||
source = RecordedRavnoves00Source(
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
|
||||
from k1link.service_watchdog import (
|
||||
ConsecutiveHealthGate,
|
||||
MissionCoreSelfWatchdog,
|
||||
MissionCoreWatchdogJournal,
|
||||
MissionCoreWatchdogPolicy,
|
||||
watchdog_enabled,
|
||||
)
|
||||
|
||||
|
||||
def test_consecutive_health_gate_resets_after_recovery() -> None:
|
||||
gate = ConsecutiveHealthGate(3)
|
||||
|
||||
assert gate.observe(False) is False
|
||||
assert gate.observe(False) is False
|
||||
assert gate.observe(True) is False
|
||||
assert gate.consecutive_failures == 0
|
||||
assert gate.observe(False) is False
|
||||
assert gate.observe(False) is False
|
||||
assert gate.observe(False) is True
|
||||
|
||||
|
||||
def test_self_watchdog_escalates_a_persistently_unhealthy_process(tmp_path: Path) -> None:
|
||||
requested = Event()
|
||||
forced = Event()
|
||||
journal_path = tmp_path / "watchdog.jsonl"
|
||||
watchdog = MissionCoreSelfWatchdog(
|
||||
tmp_path,
|
||||
policy=MissionCoreWatchdogPolicy(
|
||||
startup_grace_seconds=0.01,
|
||||
probe_interval_seconds=0.01,
|
||||
probe_timeout_seconds=0.01,
|
||||
consecutive_failure_limit=2,
|
||||
graceful_shutdown_seconds=0.02,
|
||||
),
|
||||
probe=lambda: False,
|
||||
request_shutdown=requested.set,
|
||||
force_shutdown=forced.set,
|
||||
journal=MissionCoreWatchdogJournal(journal_path),
|
||||
)
|
||||
|
||||
watchdog.start()
|
||||
|
||||
assert requested.wait(0.5)
|
||||
assert forced.wait(0.5)
|
||||
watchdog.stop()
|
||||
events = [json.loads(line)["event"] for line in journal_path.read_text().splitlines()]
|
||||
assert events == [
|
||||
"watchdog-started",
|
||||
"health-state-changed",
|
||||
"restart-requested",
|
||||
"restart-escalated",
|
||||
"watchdog-stopped",
|
||||
]
|
||||
|
||||
|
||||
def test_self_watchdog_leaves_a_healthy_process_running(tmp_path: Path) -> None:
|
||||
probed = Event()
|
||||
requested = Event()
|
||||
forced = Event()
|
||||
journal_path = tmp_path / "watchdog.jsonl"
|
||||
|
||||
def healthy_probe() -> bool:
|
||||
probed.set()
|
||||
return True
|
||||
|
||||
watchdog = MissionCoreSelfWatchdog(
|
||||
tmp_path,
|
||||
policy=MissionCoreWatchdogPolicy(
|
||||
startup_grace_seconds=0.01,
|
||||
probe_interval_seconds=0.01,
|
||||
probe_timeout_seconds=0.01,
|
||||
consecutive_failure_limit=2,
|
||||
graceful_shutdown_seconds=0.02,
|
||||
),
|
||||
probe=healthy_probe,
|
||||
request_shutdown=requested.set,
|
||||
force_shutdown=forced.set,
|
||||
journal=MissionCoreWatchdogJournal(journal_path),
|
||||
)
|
||||
|
||||
watchdog.start()
|
||||
|
||||
assert probed.wait(0.5)
|
||||
watchdog.stop()
|
||||
assert requested.is_set() is False
|
||||
assert forced.is_set() is False
|
||||
events = [json.loads(line)["event"] for line in journal_path.read_text().splitlines()]
|
||||
assert events == [
|
||||
"watchdog-started",
|
||||
"health-state-changed",
|
||||
"watchdog-stopped",
|
||||
]
|
||||
|
||||
|
||||
def test_watchdog_journal_rotates_before_exceeding_bound(tmp_path: Path) -> None:
|
||||
path = tmp_path / "watchdog.jsonl"
|
||||
journal = MissionCoreWatchdogJournal(path, max_bytes=300)
|
||||
|
||||
journal.append("first", payload="x" * 180)
|
||||
journal.append("second", payload="y" * 180)
|
||||
|
||||
assert path.is_file()
|
||||
assert path.with_name("watchdog.jsonl.1").is_file()
|
||||
assert json.loads(path.read_text())["event"] == "second"
|
||||
|
||||
|
||||
def test_watchdog_requires_exact_enable_marker() -> None:
|
||||
assert watchdog_enabled({"MISSIONCORE_SERVICE_WATCHDOG": "1"}) is True
|
||||
assert watchdog_enabled({"MISSIONCORE_SERVICE_WATCHDOG": "true"}) is False
|
||||
assert watchdog_enabled({}) is False
|
||||
Reference in New Issue
Block a user