fix(k1): stabilize repeated acquisition and live viewer recovery
This commit is contained in:
@@ -1,6 +1,15 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
import type { SceneSettings } from "../sceneSettings";
|
||||
import {
|
||||
advanceLiveReceiverWatchdog,
|
||||
initialLiveReceiverRecoveryState,
|
||||
initialLiveReceiverWatchdogState,
|
||||
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
||||
requestLiveReceiverRecovery,
|
||||
} from "../core/observation/liveReceiverWatchdog";
|
||||
import { postLiveViewerDiagnostic } from "../core/observation/liveViewerDiagnostics";
|
||||
import type { LiveViewerFailureStage } from "../core/observation/liveViewerDiagnostics";
|
||||
import type { RecordedAdmissionPhase } from "../core/observation/recordedSessionAdmission";
|
||||
|
||||
export type RerunViewportStatus = "idle" | "loading" | "ready" | "error";
|
||||
@@ -66,6 +75,8 @@ export interface RerunViewportProps {
|
||||
sourceUrl: string;
|
||||
recordedArtifact?: RecordedRrdArtifactDescriptor | null;
|
||||
followLive?: boolean;
|
||||
liveActivitySequence?: number | null;
|
||||
liveStreamId?: string | null;
|
||||
autoplayWhenReady?: boolean;
|
||||
presentationGate?: RecordedAdmissionPhase;
|
||||
expectedTimelineStartSeconds?: number;
|
||||
@@ -796,6 +807,8 @@ export function RerunViewport({
|
||||
sourceUrl,
|
||||
recordedArtifact = null,
|
||||
followLive = false,
|
||||
liveActivitySequence = null,
|
||||
liveStreamId = null,
|
||||
autoplayWhenReady = false,
|
||||
presentationGate = "ready",
|
||||
expectedTimelineStartSeconds,
|
||||
@@ -824,6 +837,11 @@ export function RerunViewport({
|
||||
const [status, setStatus] = useState<RerunViewportStatus>(sourceUrl ? "loading" : "idle");
|
||||
const [recordingBufferProgress, setRecordingBufferProgress] = useState<number | null>(null);
|
||||
const [retryNonce, setRetryNonce] = useState(0);
|
||||
const liveActivitySequenceRef = useRef<number | null>(liveActivitySequence);
|
||||
liveActivitySequenceRef.current = liveActivitySequence;
|
||||
const liveStreamIdRef = useRef<string | null>(liveStreamId);
|
||||
liveStreamIdRef.current = liveStreamId;
|
||||
const liveRecoveryRef = useRef(initialLiveReceiverRecoveryState());
|
||||
const blueprintChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
||||
const perceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
||||
const loadedPerceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
||||
@@ -849,6 +867,10 @@ export function RerunViewport({
|
||||
recordedArtifact !== null,
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
liveRecoveryRef.current = initialLiveReceiverRecoveryState();
|
||||
}, [followLive, liveStreamId, sourceUrl]);
|
||||
|
||||
useEffect(() => {
|
||||
const normalizedSource = sourceUrl.trim();
|
||||
const host = hostRef.current;
|
||||
@@ -889,6 +911,7 @@ export function RerunViewport({
|
||||
let disposed = false;
|
||||
let disposeViewer: (() => void) | undefined;
|
||||
let recordingOpenTimer: number | undefined;
|
||||
let liveRecordingDiscoveryTimer: number | undefined;
|
||||
let recordedOpenWatchdog: {
|
||||
arm: () => void;
|
||||
clear: () => void;
|
||||
@@ -898,6 +921,8 @@ export function RerunViewport({
|
||||
let recordingOpened = false;
|
||||
let recordingOpenTimedOut = false;
|
||||
let viewerStartResolved = false;
|
||||
let latestLiveRangeMaxNs: number | null = null;
|
||||
let liveWatchdog = initialLiveReceiverWatchdogState();
|
||||
// The backend serves this exact digest-bound URL or rejects it with 412.
|
||||
// Presentation still waits for Rerun to decode the complete declared range.
|
||||
const artifactVerified = true;
|
||||
@@ -925,9 +950,16 @@ export function RerunViewport({
|
||||
recordingOpenTimer = undefined;
|
||||
}
|
||||
};
|
||||
const clearLiveRecordingDiscoveryTimer = () => {
|
||||
if (liveRecordingDiscoveryTimer !== undefined) {
|
||||
window.clearInterval(liveRecordingDiscoveryTimer);
|
||||
liveRecordingDiscoveryTimer = undefined;
|
||||
}
|
||||
};
|
||||
const clearRecordingTimers = () => {
|
||||
clearRecordedAdmissionWatchdog();
|
||||
clearLiveRecordingOpenTimer();
|
||||
clearLiveRecordingDiscoveryTimer();
|
||||
};
|
||||
const clearPlaybackRangeTimer = () => {
|
||||
if (playbackRangeTimer === undefined) return;
|
||||
@@ -950,11 +982,118 @@ export function RerunViewport({
|
||||
requestFrame: (callback) => window.requestAnimationFrame(callback),
|
||||
cancelFrame: (handle) => window.cancelAnimationFrame(handle),
|
||||
});
|
||||
const reportError = (message: string) => {
|
||||
const reportError = (message: string, failureStage?: LiveViewerFailureStage) => {
|
||||
if (disposed) return;
|
||||
if (followLive) {
|
||||
postLiveViewerDiagnostic({
|
||||
eventCode: "live_receiver_error",
|
||||
failureStage,
|
||||
streamId: liveStreamIdRef.current,
|
||||
backendActivitySequence: liveActivitySequenceRef.current,
|
||||
viewerRangeMaxNs: latestLiveRangeMaxNs,
|
||||
recoveryAttempt: liveRecoveryRef.current.attempts || null,
|
||||
});
|
||||
}
|
||||
setStatus("error");
|
||||
onStatusChange?.("error", message);
|
||||
};
|
||||
const requestLiveRecovery = (
|
||||
failureStage: LiveViewerFailureStage,
|
||||
stalledForMs = 0,
|
||||
emitErrorEvent = true,
|
||||
) => {
|
||||
if (!followLive || disposed) return false;
|
||||
if (emitErrorEvent) {
|
||||
postLiveViewerDiagnostic({
|
||||
eventCode: "live_receiver_error",
|
||||
failureStage,
|
||||
streamId: liveStreamIdRef.current,
|
||||
backendActivitySequence: liveActivitySequenceRef.current,
|
||||
viewerRangeMaxNs: latestLiveRangeMaxNs,
|
||||
stalledForMs: Math.round(stalledForMs) || null,
|
||||
recoveryAttempt: liveRecoveryRef.current.attempts || null,
|
||||
});
|
||||
}
|
||||
const recovery = requestLiveReceiverRecovery(liveRecoveryRef.current);
|
||||
liveRecoveryRef.current = recovery.state;
|
||||
if (recovery.signal === "exhausted") {
|
||||
postLiveViewerDiagnostic({
|
||||
eventCode: "live_receiver_recovery_exhausted",
|
||||
failureStage,
|
||||
streamId: liveStreamIdRef.current,
|
||||
backendActivitySequence: liveActivitySequenceRef.current,
|
||||
viewerRangeMaxNs: latestLiveRangeMaxNs,
|
||||
stalledForMs: Math.round(stalledForMs) || null,
|
||||
recoveryAttempt: LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
||||
});
|
||||
setStatus("error");
|
||||
onStatusChange?.(
|
||||
"error",
|
||||
"Живой поток продолжает поступать, но визуализатор не восстановился после трёх переподключений.",
|
||||
);
|
||||
return true;
|
||||
}
|
||||
postLiveViewerDiagnostic({
|
||||
eventCode: "live_receiver_restart_requested",
|
||||
failureStage,
|
||||
streamId: liveStreamIdRef.current,
|
||||
backendActivitySequence: liveActivitySequenceRef.current,
|
||||
viewerRangeMaxNs: latestLiveRangeMaxNs,
|
||||
stalledForMs: Math.round(stalledForMs) || null,
|
||||
recoveryAttempt: recovery.attempt,
|
||||
});
|
||||
setStatus("loading");
|
||||
onStatusChange?.(
|
||||
"loading",
|
||||
"Живой визуализатор переподключается к продолжающемуся потоку.",
|
||||
);
|
||||
disposeViewer?.();
|
||||
setRetryNonce((nonce) => nonce + 1);
|
||||
return true;
|
||||
};
|
||||
const observeLiveReceiver = (viewerRangeMaxNs: number | null) => {
|
||||
if (!followLive || disposed) return;
|
||||
latestLiveRangeMaxNs = viewerRangeMaxNs;
|
||||
const receiverOpenedAfterRecovery =
|
||||
liveRecoveryRef.current.awaitingRecovery &&
|
||||
liveWatchdog.lastViewerRangeMaxNs === null &&
|
||||
viewerRangeMaxNs !== null;
|
||||
const observed = advanceLiveReceiverWatchdog(liveWatchdog, {
|
||||
nowMs: Date.now(),
|
||||
backendActivitySequence: liveActivitySequenceRef.current,
|
||||
viewerRangeMaxNs,
|
||||
});
|
||||
liveWatchdog = observed.state;
|
||||
if (
|
||||
(receiverOpenedAfterRecovery || observed.signal === "receiver-advanced") &&
|
||||
liveRecoveryRef.current.awaitingRecovery
|
||||
) {
|
||||
postLiveViewerDiagnostic({
|
||||
eventCode: "live_receiver_recovered",
|
||||
streamId: liveStreamIdRef.current,
|
||||
backendActivitySequence: liveActivitySequenceRef.current,
|
||||
viewerRangeMaxNs,
|
||||
recoveryAttempt: liveRecoveryRef.current.attempts,
|
||||
});
|
||||
liveRecoveryRef.current = initialLiveReceiverRecoveryState();
|
||||
return;
|
||||
}
|
||||
if (observed.signal !== "stalled") return;
|
||||
|
||||
postLiveViewerDiagnostic({
|
||||
eventCode: "live_receiver_stalled",
|
||||
failureStage: "receiver-stalled",
|
||||
streamId: liveStreamIdRef.current,
|
||||
backendActivitySequence: liveActivitySequenceRef.current,
|
||||
viewerRangeMaxNs,
|
||||
stalledForMs: Math.round(observed.stalledForMs),
|
||||
recoveryAttempt: Math.min(
|
||||
liveRecoveryRef.current.attempts + 1,
|
||||
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
||||
),
|
||||
});
|
||||
requestLiveRecovery("receiver-stalled", observed.stalledForMs, false);
|
||||
};
|
||||
|
||||
host.replaceChildren();
|
||||
appliedPointColorKeyRef.current = null;
|
||||
@@ -1016,7 +1155,7 @@ export function RerunViewport({
|
||||
viewer.stop();
|
||||
} catch {
|
||||
// A partially initialized WASM handle can already be gone after a
|
||||
// startup failure. The host still has to be cleared below.
|
||||
// startup failure.
|
||||
}
|
||||
host.replaceChildren();
|
||||
};
|
||||
@@ -1034,14 +1173,20 @@ export function RerunViewport({
|
||||
});
|
||||
}
|
||||
|
||||
unsubscribers.push(
|
||||
viewer.on("recording_open", (event) => {
|
||||
const admitRecording = (event: {
|
||||
application_id: string;
|
||||
recording_id: string;
|
||||
}) => {
|
||||
if (
|
||||
disposed ||
|
||||
recordingOpened ||
|
||||
(isRecordedSource && event.application_id !== "nodedc_mission_core_recorded")
|
||||
) return;
|
||||
recordingOpened = true;
|
||||
if (!isRecordedSource) {
|
||||
clearLiveRecordingOpenTimer();
|
||||
clearLiveRecordingDiscoveryTimer();
|
||||
}
|
||||
if (
|
||||
recordedBlueprintUrl &&
|
||||
event.application_id === "nodedc_mission_core_recorded" &&
|
||||
@@ -1060,7 +1205,6 @@ export function RerunViewport({
|
||||
setPerceptionChannelRevision((revision) => revision + 1);
|
||||
}
|
||||
}
|
||||
if (!isRecordedSource) clearLiveRecordingOpenTimer();
|
||||
// Durable Mission Core recordings use a zero-based session clock.
|
||||
// Keeping wall-clock capture_time as a secondary RRD timeline is
|
||||
// useful for audit, but the operator scrubber must remain a small,
|
||||
@@ -1144,6 +1288,7 @@ export function RerunViewport({
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
observeLiveReceiver(rangeNs?.max ?? null);
|
||||
const recordedBuffer = recordedPlaybackBufferState(
|
||||
rangeNs,
|
||||
expectedTimelineEndSeconds,
|
||||
@@ -1225,8 +1370,31 @@ export function RerunViewport({
|
||||
clearPlaybackRangeTimer();
|
||||
publishBufferedState();
|
||||
playbackRangeTimer = window.setInterval(publishBufferedState, 500);
|
||||
}),
|
||||
};
|
||||
unsubscribers.push(
|
||||
viewer.on("recording_open", admitRecording),
|
||||
);
|
||||
const discoverActiveLiveRecording = () => {
|
||||
if (isRecordedSource || disposed || recordingOpened || !viewer.ready) return;
|
||||
try {
|
||||
const recordingId = viewer.get_active_recording_id();
|
||||
if (!recordingId) return;
|
||||
// Rerun 0.34.1 may ingest an SDK gRPC store without forwarding its
|
||||
// recording_open event to the JavaScript wrapper. The active store
|
||||
// is the authoritative fallback and avoids hiding a ready canvas.
|
||||
postLiveViewerDiagnostic({
|
||||
eventCode: "live_receiver_active_store_admitted",
|
||||
streamId: liveStreamIdRef.current,
|
||||
backendActivitySequence: liveActivitySequenceRef.current,
|
||||
});
|
||||
admitRecording({
|
||||
application_id: "nodedc_mission_core_spatial",
|
||||
recording_id: recordingId,
|
||||
});
|
||||
} catch {
|
||||
// The native receiver is still opening; the bounded timer retries.
|
||||
}
|
||||
};
|
||||
|
||||
unsubscribers.push(
|
||||
viewer.on("time_update", (event) => {
|
||||
@@ -1316,25 +1484,34 @@ export function RerunViewport({
|
||||
setPerceptionChannelRevision((revision) => revision + 1);
|
||||
}
|
||||
|
||||
if (!recordingOpened && !isRecordedSource) {
|
||||
if (!isRecordedSource) {
|
||||
discoverActiveLiveRecording();
|
||||
if (!recordingOpened) {
|
||||
liveRecordingDiscoveryTimer = window.setInterval(
|
||||
discoverActiveLiveRecording,
|
||||
100,
|
||||
);
|
||||
recordingOpenTimer = window.setTimeout(() => {
|
||||
recordingOpenTimedOut = true;
|
||||
disposeViewer?.();
|
||||
reportError("Визуализатор запущен, но поток записи не открылся.");
|
||||
requestLiveRecovery("recording-open-timeout");
|
||||
}, 12_000);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
disposeViewer();
|
||||
if (!recordingOpenTimedOut && !disposed) {
|
||||
reportError(isRecordedSource
|
||||
const message = isRecordedSource
|
||||
? "Запись не прошла атомарную проверку целостности."
|
||||
: "Не удалось запустить встроенный визуализатор.");
|
||||
: "Не удалось запустить встроенный визуализатор.";
|
||||
if (!isRecordedSource && requestLiveRecovery("viewer-start")) return;
|
||||
reportError(message);
|
||||
}
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
if (disposed) return;
|
||||
disposeViewer?.();
|
||||
if (requestLiveRecovery("module-load")) return;
|
||||
reportError("Не удалось загрузить модуль визуализатора.");
|
||||
});
|
||||
|
||||
@@ -1355,6 +1532,7 @@ export function RerunViewport({
|
||||
expectedTimelineStartSeconds,
|
||||
followLive,
|
||||
initialPlaybackStartSeconds,
|
||||
liveStreamId,
|
||||
onPlaybackChange,
|
||||
onPlaybackControllerChange,
|
||||
onSelectionChange,
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
export interface LiveReceiverWatchdogState {
|
||||
lastBackendActivitySequence: number | null;
|
||||
lastViewerRangeMaxNs: number | null;
|
||||
staticRangeBackendAdvances: number;
|
||||
stalledSinceMs: number | null;
|
||||
stallReported: boolean;
|
||||
}
|
||||
|
||||
export type LiveReceiverWatchdogSignal = "none" | "receiver-advanced" | "stalled";
|
||||
|
||||
export interface LiveReceiverWatchdogResult {
|
||||
state: LiveReceiverWatchdogState;
|
||||
signal: LiveReceiverWatchdogSignal;
|
||||
stalledForMs: number;
|
||||
}
|
||||
|
||||
export interface LiveReceiverRecoveryState {
|
||||
attempts: number;
|
||||
awaitingRecovery: boolean;
|
||||
}
|
||||
|
||||
export type LiveReceiverRecoverySignal = "retry" | "exhausted";
|
||||
|
||||
export interface LiveReceiverRecoveryResult {
|
||||
state: LiveReceiverRecoveryState;
|
||||
signal: LiveReceiverRecoverySignal;
|
||||
attempt: number;
|
||||
}
|
||||
|
||||
export const LIVE_RECEIVER_STALL_THRESHOLD_MS = 5_000;
|
||||
export const LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS = 3;
|
||||
|
||||
export function initialLiveReceiverWatchdogState(): LiveReceiverWatchdogState {
|
||||
return {
|
||||
lastBackendActivitySequence: null,
|
||||
lastViewerRangeMaxNs: null,
|
||||
staticRangeBackendAdvances: 0,
|
||||
stalledSinceMs: null,
|
||||
stallReported: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function initialLiveReceiverRecoveryState(): LiveReceiverRecoveryState {
|
||||
return {
|
||||
attempts: 0,
|
||||
awaitingRecovery: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Bound viewer-only restarts independently from scanner and acquisition
|
||||
* lifecycle. The caller may dispose and recreate the browser receiver, but
|
||||
* must never issue START/STOP or reconnect the physical device.
|
||||
*/
|
||||
export function requestLiveReceiverRecovery(
|
||||
current: LiveReceiverRecoveryState,
|
||||
maxAttempts = LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
||||
): LiveReceiverRecoveryResult {
|
||||
if (current.attempts >= maxAttempts) {
|
||||
return {
|
||||
state: {
|
||||
attempts: current.attempts,
|
||||
awaitingRecovery: false,
|
||||
},
|
||||
signal: "exhausted",
|
||||
attempt: current.attempts,
|
||||
};
|
||||
}
|
||||
const attempt = current.attempts + 1;
|
||||
return {
|
||||
state: {
|
||||
attempts: attempt,
|
||||
awaitingRecovery: true,
|
||||
},
|
||||
signal: "retry",
|
||||
attempt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect a disposable receiver stall only when the backend frame counter keeps
|
||||
* advancing while Rerun's visible time range does not. Scanner commands are
|
||||
* deliberately absent from this state machine.
|
||||
*/
|
||||
export function advanceLiveReceiverWatchdog(
|
||||
current: LiveReceiverWatchdogState,
|
||||
sample: {
|
||||
nowMs: number;
|
||||
backendActivitySequence: number | null;
|
||||
viewerRangeMaxNs: number | null;
|
||||
},
|
||||
thresholdMs = LIVE_RECEIVER_STALL_THRESHOLD_MS,
|
||||
): LiveReceiverWatchdogResult {
|
||||
const backendSequence = Number.isSafeInteger(sample.backendActivitySequence) &&
|
||||
(sample.backendActivitySequence ?? -1) >= 0
|
||||
? sample.backendActivitySequence
|
||||
: null;
|
||||
const viewerRange = Number.isFinite(sample.viewerRangeMaxNs) &&
|
||||
(sample.viewerRangeMaxNs ?? -1) >= 0
|
||||
? sample.viewerRangeMaxNs
|
||||
: null;
|
||||
const state = { ...current };
|
||||
|
||||
if (state.lastBackendActivitySequence === null && state.lastViewerRangeMaxNs === null) {
|
||||
state.lastBackendActivitySequence = backendSequence;
|
||||
state.lastViewerRangeMaxNs = viewerRange;
|
||||
return { state, signal: "none", stalledForMs: 0 };
|
||||
}
|
||||
|
||||
const backendAdvanced = backendSequence !== null &&
|
||||
(state.lastBackendActivitySequence === null ||
|
||||
backendSequence > state.lastBackendActivitySequence);
|
||||
const receiverAdvanced = viewerRange !== null &&
|
||||
(state.lastViewerRangeMaxNs === null || viewerRange > state.lastViewerRangeMaxNs);
|
||||
|
||||
if (backendSequence !== null) {
|
||||
state.lastBackendActivitySequence = Math.max(
|
||||
backendSequence,
|
||||
state.lastBackendActivitySequence ?? 0,
|
||||
);
|
||||
}
|
||||
if (viewerRange !== null) {
|
||||
state.lastViewerRangeMaxNs = Math.max(viewerRange, state.lastViewerRangeMaxNs ?? 0);
|
||||
}
|
||||
|
||||
if (receiverAdvanced) {
|
||||
const hadStaticBackendProgress = state.stalledSinceMs !== null;
|
||||
state.staticRangeBackendAdvances = 0;
|
||||
state.stalledSinceMs = null;
|
||||
state.stallReported = false;
|
||||
return {
|
||||
state,
|
||||
signal: hadStaticBackendProgress ? "receiver-advanced" : "none",
|
||||
stalledForMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (backendAdvanced) {
|
||||
state.staticRangeBackendAdvances += 1;
|
||||
if (state.staticRangeBackendAdvances >= 2 && state.stalledSinceMs === null) {
|
||||
state.stalledSinceMs = sample.nowMs;
|
||||
}
|
||||
}
|
||||
const stalledForMs = state.stalledSinceMs === null
|
||||
? 0
|
||||
: Math.max(0, sample.nowMs - state.stalledSinceMs);
|
||||
if (
|
||||
!state.stallReported &&
|
||||
state.staticRangeBackendAdvances >= 2 &&
|
||||
stalledForMs >= thresholdMs
|
||||
) {
|
||||
state.stallReported = true;
|
||||
return { state, signal: "stalled", stalledForMs };
|
||||
}
|
||||
return { state, signal: "none", stalledForMs };
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
export type LiveViewerDiagnosticEventCode =
|
||||
| "live_receiver_stalled"
|
||||
| "live_receiver_restart_requested"
|
||||
| "live_receiver_recovered"
|
||||
| "live_receiver_recovery_exhausted"
|
||||
| "live_receiver_active_store_admitted"
|
||||
| "live_receiver_error";
|
||||
|
||||
export type LiveViewerFailureStage =
|
||||
| "recording-open-timeout"
|
||||
| "viewer-start"
|
||||
| "module-load"
|
||||
| "receiver-stalled";
|
||||
|
||||
export interface LiveViewerDiagnostic {
|
||||
eventCode: LiveViewerDiagnosticEventCode;
|
||||
failureStage?: LiveViewerFailureStage | null;
|
||||
streamId?: string | null;
|
||||
backendActivitySequence?: number | null;
|
||||
viewerRangeMaxNs?: number | null;
|
||||
stalledForMs?: number | null;
|
||||
recoveryAttempt?: number | null;
|
||||
}
|
||||
|
||||
const SAFE_STREAM_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
|
||||
function safeInteger(value: number | null | undefined): number | undefined {
|
||||
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0
|
||||
? value
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export function postLiveViewerDiagnostic(event: LiveViewerDiagnostic): void {
|
||||
const streamId = event.streamId?.trim();
|
||||
const body = {
|
||||
schema_version: "missioncore.live-viewer-diagnostic/v1",
|
||||
event_code: event.eventCode,
|
||||
...(event.failureStage ? { failure_stage: event.failureStage } : {}),
|
||||
...(streamId && SAFE_STREAM_ID.test(streamId) ? { stream_id: streamId } : {}),
|
||||
...(safeInteger(event.backendActivitySequence) === undefined
|
||||
? {}
|
||||
: { backend_activity_sequence: safeInteger(event.backendActivitySequence) }),
|
||||
...(safeInteger(event.viewerRangeMaxNs) === undefined
|
||||
? {}
|
||||
: { viewer_range_max_ns: safeInteger(event.viewerRangeMaxNs) }),
|
||||
...(safeInteger(event.stalledForMs) === undefined
|
||||
? {}
|
||||
: { stalled_for_ms: safeInteger(event.stalledForMs) }),
|
||||
...(safeInteger(event.recoveryAttempt) === undefined
|
||||
? {}
|
||||
: { recovery_attempt: safeInteger(event.recoveryAttempt) }),
|
||||
};
|
||||
void fetch("/api/v1/viewer/live-diagnostics", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
keepalive: true,
|
||||
}).catch(() => {
|
||||
// Diagnostics must never interfere with the live receiver recovery path.
|
||||
});
|
||||
}
|
||||
@@ -28,6 +28,7 @@ export interface ViewerSettings {
|
||||
}
|
||||
|
||||
export interface StreamMetrics {
|
||||
publishedFrameCount?: number | null;
|
||||
latencyMs?: number | null;
|
||||
frameRateHz?: number | null;
|
||||
pointCount?: number | null;
|
||||
|
||||
@@ -499,7 +499,8 @@ function SpatialWorkspace({
|
||||
<RerunViewport
|
||||
sourceUrl={sourceUrl}
|
||||
recordedArtifact={recordedSource ? recordedReplay : null}
|
||||
followLive={!recordedSource && state?.sourceMode === "live"}
|
||||
followLive={!recordedReplay && streamActive}
|
||||
liveActivitySequence={metrics?.publishedFrameCount} liveStreamId={state?.spatialSource?.id}
|
||||
autoplayWhenReady={recordedSource}
|
||||
presentationGate={recordedSessionGate}
|
||||
expectedTimelineStartSeconds={recordedSource
|
||||
|
||||
@@ -439,6 +439,11 @@ test("live source is confirmed only by an acquiring acquisition", () => {
|
||||
assert.equal(lifecycle.sourceStatusLabel(waiting), "Ожидание реальных данных");
|
||||
assert.equal(lifecycle.isConfirmedLiveState(acquiring), true);
|
||||
assert.equal(lifecycle.confirmedRuntimeSourceMode(acquiring), "live");
|
||||
assert.equal(lifecycle.liveStartPlan(acquiring), "already-running");
|
||||
assert.equal(
|
||||
lifecycle.controlSessionEntryPlan("scanning", false, false),
|
||||
"continue",
|
||||
);
|
||||
});
|
||||
|
||||
test("prepared acquisition resumes without another prepare and remains recoverable", () => {
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { after, before, test } from "node:test";
|
||||
|
||||
import { createServer } from "vite";
|
||||
|
||||
let server;
|
||||
let advanceLiveReceiverWatchdog;
|
||||
let initialLiveReceiverWatchdogState;
|
||||
let initialLiveReceiverRecoveryState;
|
||||
let requestLiveReceiverRecovery;
|
||||
|
||||
before(async () => {
|
||||
server = await createServer({
|
||||
appType: "custom",
|
||||
logLevel: "silent",
|
||||
server: { middlewareMode: true },
|
||||
});
|
||||
({
|
||||
advanceLiveReceiverWatchdog,
|
||||
initialLiveReceiverRecoveryState,
|
||||
initialLiveReceiverWatchdogState,
|
||||
requestLiveReceiverRecovery,
|
||||
} = await server.ssrLoadModule("/src/core/observation/liveReceiverWatchdog.ts"));
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await server?.close();
|
||||
});
|
||||
|
||||
test("watchdog ignores healthy live progress", () => {
|
||||
let state = initialLiveReceiverWatchdogState();
|
||||
for (let index = 0; index < 20; index += 1) {
|
||||
const observed = advanceLiveReceiverWatchdog(state, {
|
||||
nowMs: index * 500,
|
||||
backendActivitySequence: index * 2,
|
||||
viewerRangeMaxNs: index * 1_000_000_000,
|
||||
});
|
||||
state = observed.state;
|
||||
assert.notEqual(observed.signal, "stalled");
|
||||
}
|
||||
});
|
||||
|
||||
test("watchdog detects a frozen receiver while backend publication advances", () => {
|
||||
let state = initialLiveReceiverWatchdogState();
|
||||
let signal = "none";
|
||||
let stalledForMs = 0;
|
||||
for (let index = 0; index <= 13; index += 1) {
|
||||
const observed = advanceLiveReceiverWatchdog(state, {
|
||||
nowMs: index * 500,
|
||||
backendActivitySequence: 100 + index,
|
||||
viewerRangeMaxNs: 42_000_000_000,
|
||||
});
|
||||
state = observed.state;
|
||||
signal = observed.signal === "stalled" ? observed.signal : signal;
|
||||
stalledForMs = Math.max(stalledForMs, observed.stalledForMs);
|
||||
}
|
||||
assert.equal(signal, "stalled");
|
||||
assert.ok(stalledForMs >= 5_000);
|
||||
});
|
||||
|
||||
test("one backend increment at stream finalization does not create a false stall", () => {
|
||||
let state = initialLiveReceiverWatchdogState();
|
||||
state = advanceLiveReceiverWatchdog(state, {
|
||||
nowMs: 0,
|
||||
backendActivitySequence: 10,
|
||||
viewerRangeMaxNs: 20_000_000_000,
|
||||
}).state;
|
||||
state = advanceLiveReceiverWatchdog(state, {
|
||||
nowMs: 500,
|
||||
backendActivitySequence: 11,
|
||||
viewerRangeMaxNs: 20_000_000_000,
|
||||
}).state;
|
||||
const observed = advanceLiveReceiverWatchdog(state, {
|
||||
nowMs: 10_000,
|
||||
backendActivitySequence: 11,
|
||||
viewerRangeMaxNs: 20_000_000_000,
|
||||
});
|
||||
assert.equal(observed.signal, "none");
|
||||
});
|
||||
|
||||
test("startup failures request only three bounded viewer restarts", () => {
|
||||
let state = initialLiveReceiverRecoveryState();
|
||||
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
||||
const recovery = requestLiveReceiverRecovery(state);
|
||||
assert.equal(recovery.signal, "retry");
|
||||
assert.equal(recovery.attempt, attempt);
|
||||
assert.equal(recovery.state.awaitingRecovery, true);
|
||||
state = recovery.state;
|
||||
}
|
||||
const exhausted = requestLiveReceiverRecovery(state);
|
||||
assert.equal(exhausted.signal, "exhausted");
|
||||
assert.equal(exhausted.attempt, 3);
|
||||
assert.equal(exhausted.state.awaitingRecovery, false);
|
||||
});
|
||||
@@ -129,7 +129,11 @@ test("recorded RRD bytes are never split across LogChannel.send_rrd calls", asyn
|
||||
assert.match(source, /viewer\.start\(\s*rerunViewerInitialSource\(resolvedSource\)/s);
|
||||
assert.match(
|
||||
source,
|
||||
/recordingOpened = true;[\s\S]*if \(!isRecordedSource\) clearLiveRecordingOpenTimer\(\);/,
|
||||
/recordingOpened = true;[\s\S]*clearLiveRecordingOpenTimer\(\);[\s\S]*clearLiveRecordingDiscoveryTimer\(\);/,
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/viewer\.get_active_recording_id\(\)[\s\S]*eventCode: "live_receiver_active_store_admitted"[\s\S]*admitRecording\(\{[\s\S]*application_id: "nodedc_mission_core_spatial"/,
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
@@ -141,6 +145,14 @@ test("recorded RRD bytes are never split across LogChannel.send_rrd calls", asyn
|
||||
);
|
||||
});
|
||||
|
||||
test("raw replay exercises the same streaming receiver lifecycle as a live scan", async () => {
|
||||
const source = await readFile(
|
||||
new URL("../src/workspaces/Workspaces.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
assert.match(source, /followLive=\{!recordedReplay && streamActive\}/);
|
||||
});
|
||||
|
||||
test("the complete vendor canvas host is hidden during partial and failed admission", async () => {
|
||||
const css = await readFile(new URL("../src/styles/spatial.css", import.meta.url), "utf8");
|
||||
assert.match(
|
||||
|
||||
@@ -58,6 +58,30 @@ export interface XgridsDeviceSession {
|
||||
connectivity?: "unknown" | "offline" | "connecting" | "connected" | "degraded";
|
||||
}
|
||||
|
||||
export interface XgridsConnectionVerification {
|
||||
status?:
|
||||
| "not-probed"
|
||||
| "configured"
|
||||
| "live-address-observed"
|
||||
| "reachable"
|
||||
| "recovered"
|
||||
| "unreachable";
|
||||
lease_state?: "disconnected" | "configured" | "reachable";
|
||||
lease_generation?: number;
|
||||
endpoint_validation?: string | null;
|
||||
network_reachability?:
|
||||
| "unknown"
|
||||
| "not-probed"
|
||||
| "reachable"
|
||||
| "degraded"
|
||||
| "unreachable";
|
||||
address_changed?: boolean;
|
||||
previous_address_present?: boolean;
|
||||
write_performed?: boolean;
|
||||
observed_at?: string | null;
|
||||
reason_code?: string | null;
|
||||
}
|
||||
|
||||
export interface XgridsCompatibilityState {
|
||||
profile_id?: string | null;
|
||||
decision?: "compatible" | "limited" | "unknown" | "incompatible";
|
||||
@@ -167,6 +191,23 @@ export interface XgridsApplicationControlSession {
|
||||
expected?: Record<string, unknown>;
|
||||
observed?: Record<string, unknown>;
|
||||
} | null;
|
||||
status_reconciliation?: {
|
||||
device_session_state?: string;
|
||||
device_project_bound?: boolean;
|
||||
system_error_code?: number | null;
|
||||
decision?: "safe-explicit-prestart-retry";
|
||||
automatic_retry?: false;
|
||||
} | null;
|
||||
network_change_admissible?: boolean;
|
||||
network_change_reconciliation?: {
|
||||
device_session_state?: "scan_stopping";
|
||||
device_project_bound?: true;
|
||||
system_error_code?: null;
|
||||
stop_complete?: true;
|
||||
standby_confirmed?: false;
|
||||
decision?: "explicit-network-change-only-after-acknowledged-stop";
|
||||
automatic_retry?: false;
|
||||
} | null;
|
||||
safe_to_retry?: boolean;
|
||||
} | null;
|
||||
dialogue?: Record<string, unknown> | null;
|
||||
@@ -220,6 +261,8 @@ export interface XgridsOperation {
|
||||
}
|
||||
|
||||
export interface XgridsK1Metrics {
|
||||
pcl_frames?: number | null;
|
||||
pose_frames?: number | null;
|
||||
mqtt_to_decode_ms?: number | null;
|
||||
decode_ms?: number | null;
|
||||
publish_ms?: number | null;
|
||||
@@ -308,6 +351,7 @@ export interface XgridsK1State {
|
||||
application_control_session?: XgridsApplicationControlSession | null;
|
||||
device_ref?: XgridsDeviceRef | null;
|
||||
device_session?: XgridsDeviceSession | null;
|
||||
connection_verification?: XgridsConnectionVerification | null;
|
||||
acquisition?: XgridsAcquisition | null;
|
||||
operations?: XgridsOperation[];
|
||||
last_operation?: XgridsOperation | null;
|
||||
|
||||
@@ -104,6 +104,14 @@ function normalizeState(
|
||||
viewerSettings: state.viewer_settings,
|
||||
sourceMode: confirmedRuntimeSourceMode(state),
|
||||
metrics: {
|
||||
publishedFrameCount: (
|
||||
Number.isSafeInteger(metrics?.pcl_frames) &&
|
||||
Number.isSafeInteger(metrics?.pose_frames) &&
|
||||
(metrics?.pcl_frames ?? -1) >= 0 &&
|
||||
(metrics?.pose_frames ?? -1) >= 0
|
||||
)
|
||||
? (metrics?.pcl_frames ?? 0) + (metrics?.pose_frames ?? 0)
|
||||
: null,
|
||||
latencyMs: pipelineLatency(metrics),
|
||||
frameRateHz: finiteMetric(metrics?.frame_rate ?? metrics?.frame_rate_hz),
|
||||
pointCount: finiteMetric(metrics?.point_count),
|
||||
|
||||
@@ -70,6 +70,8 @@ function controlFailure(state: XgridsK1State): ApiError {
|
||||
"Сканер не подтвердил канонические MQTT-подписки.",
|
||||
mqtt_response_timeout:
|
||||
"Ожидаемый ответ сканера не пришёл до безопасной границы ожидания.",
|
||||
mqtt_network_loop_failed:
|
||||
"Локальный MQTT-клиент потерял управляющее соединение со сканером.",
|
||||
response_identity_decode_failed:
|
||||
"Ответ сканера не удалось безопасно разобрать и привязать к операции.",
|
||||
modeling_response_decode_failed:
|
||||
@@ -124,7 +126,9 @@ function controlFailure(state: XgridsK1State): ApiError {
|
||||
? "Команды START и STOP не отправлялись."
|
||||
: "Факт отправки START или STOP диагностически не подтверждён; повтор запрещён.";
|
||||
const retryStatus = failure?.safe_to_retry
|
||||
? "Новая попытка возможна только отдельным нажатием оператора."
|
||||
? failure.status_reconciliation?.decision === "safe-explicit-prestart-retry"
|
||||
? "Живой статус READY без связанного проекта подтверждён; новая попытка возможна только отдельным нажатием оператора."
|
||||
: "Новая попытка возможна только отдельным нажатием оператора."
|
||||
: "Повтор заблокирован до ручной проверки состояния.";
|
||||
const exchanges = typeof failure?.publish_attempts === "number"
|
||||
? `MQTT-публикаций до остановки: ${failure.publish_attempts}.`
|
||||
|
||||
@@ -396,6 +396,11 @@ class LivePerceptionIngress:
|
||||
if self._session_id == session_id:
|
||||
return
|
||||
raise RuntimeError("another live perception session is active")
|
||||
# Queued items belong to one acquisition. A worker may be absent
|
||||
# while a session ends, so clear bounded leftovers before the next
|
||||
# session can become visible to that worker.
|
||||
for queue in self._queues.values():
|
||||
queue.items.clear()
|
||||
self._session_id = session_id
|
||||
self._active = True
|
||||
self._publish_locked(
|
||||
@@ -681,9 +686,7 @@ class LiveSensorSynchronizer:
|
||||
or retention_seconds <= 0
|
||||
):
|
||||
raise ValueError("live sensor synchronizer bounds are invalid")
|
||||
self._maximum_lidar_camera_delta_ns = round(
|
||||
maximum_lidar_camera_delta_ms * 1_000_000
|
||||
)
|
||||
self._maximum_lidar_camera_delta_ns = round(maximum_lidar_camera_delta_ms * 1_000_000)
|
||||
self._maximum_pose_point_delta_ns = round(maximum_pose_point_delta_ms * 1_000_000)
|
||||
self._capacity = capacity_per_modality
|
||||
self._retention_ns = round(retention_seconds * 1_000_000_000)
|
||||
@@ -801,8 +804,7 @@ class LiveSensorSynchronizer:
|
||||
newest = values[-1].context.captured_at_epoch_ns
|
||||
oldest_allowed = newest - self._retention_ns
|
||||
while values and (
|
||||
len(values) > self._capacity
|
||||
or values[0].context.captured_at_epoch_ns < oldest_allowed
|
||||
len(values) > self._capacity or values[0].context.captured_at_epoch_ns < oldest_allowed
|
||||
):
|
||||
values.popleft()
|
||||
removed += 1
|
||||
@@ -837,9 +839,7 @@ class WorldStateProjector:
|
||||
if velocity_history_limit_s <= 0:
|
||||
raise ValueError("velocity history limit must be positive")
|
||||
self._velocity_history_limit_s = velocity_history_limit_s
|
||||
self._track_history: dict[
|
||||
int, deque[tuple[float, tuple[float, float, float]]]
|
||||
] = {}
|
||||
self._track_history: dict[int, deque[tuple[float, tuple[float, float, float]]]] = {}
|
||||
|
||||
def project(
|
||||
self,
|
||||
@@ -950,9 +950,7 @@ class WorldStateProjector:
|
||||
math.sqrt(
|
||||
sum(
|
||||
(observed - expected) ** 2
|
||||
for observed, expected in zip(
|
||||
observed_center, predicted, strict=True
|
||||
)
|
||||
for observed, expected in zip(observed_center, predicted, strict=True)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -163,13 +163,18 @@ async def read_wifi_status_once(
|
||||
device_macos_uuid: str,
|
||||
*,
|
||||
timeout_seconds: float = 20.0,
|
||||
rediscover: bool = False,
|
||||
) -> WifiStatusReadResult:
|
||||
"""Read the K1's current DHCP status over BLE without writing a characteristic."""
|
||||
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("timeout_seconds must be positive")
|
||||
async with asyncio.timeout(timeout_seconds + 5.0):
|
||||
device = discovered_device(device_macos_uuid)
|
||||
# A CoreBluetooth handle retained by an earlier scan is an optimization,
|
||||
# not durable connection state. Recovery after sleep, Wi-Fi transition,
|
||||
# or a completed provisioning GATT session must rediscover the device
|
||||
# instead of repeatedly opening a stale handle.
|
||||
device = None if rediscover else discovered_device(device_macos_uuid)
|
||||
if device is None:
|
||||
device = await BleakScanner.find_device_by_address(
|
||||
device_macos_uuid,
|
||||
@@ -183,9 +188,7 @@ async def read_wifi_status_once(
|
||||
|
||||
async with BleakClient(device, timeout=timeout_seconds, pair=False) as client:
|
||||
service = client.services.get_service(SERVICE_UUID)
|
||||
status_characteristic = client.services.get_characteristic(
|
||||
STATUS_CHARACTERISTIC_UUID
|
||||
)
|
||||
status_characteristic = client.services.get_characteristic(STATUS_CHARACTERISTIC_UUID)
|
||||
if service is None:
|
||||
raise ValueError(f"Reviewed K1 service not found: {SERVICE_UUID}")
|
||||
if status_characteristic is None:
|
||||
@@ -193,9 +196,7 @@ async def read_wifi_status_once(
|
||||
f"Reviewed K1 status characteristic not found: {STATUS_CHARACTERISTIC_UUID}"
|
||||
)
|
||||
if status_characteristic.service_uuid != service.uuid:
|
||||
raise ValueError(
|
||||
"K1 status characteristic is attached to an unexpected service"
|
||||
)
|
||||
raise ValueError("K1 status characteristic is attached to an unexpected service")
|
||||
if "read" not in status_characteristic.properties:
|
||||
raise ValueError("Reviewed K1 status characteristic is not readable")
|
||||
value = bytes(await client.read_gatt_char(status_characteristic))
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -243,6 +243,9 @@ class ApplicationMqttTransportSnapshot:
|
||||
latest_device_init_ready: bool | None
|
||||
latest_system_error_code: int | None
|
||||
latest_system_error_state: str | None
|
||||
last_loop_result_code: int | None
|
||||
last_loop_result_name: str | None
|
||||
last_loop_phase: str | None
|
||||
operation_keys_consumed: int
|
||||
clean_session: bool = False
|
||||
keepalive_seconds: int = CONTROL_KEEPALIVE_SECONDS
|
||||
@@ -268,6 +271,9 @@ class ApplicationMqttTransportSnapshot:
|
||||
"latest_device_init_ready": self.latest_device_init_ready,
|
||||
"latest_system_error_code": self.latest_system_error_code,
|
||||
"latest_system_error_state": self.latest_system_error_state,
|
||||
"last_loop_result_code": self.last_loop_result_code,
|
||||
"last_loop_result_name": self.last_loop_result_name,
|
||||
"last_loop_phase": self.last_loop_phase,
|
||||
"operation_keys_consumed": self.operation_keys_consumed,
|
||||
"clean_session": self.clean_session,
|
||||
"keepalive_seconds": self.keepalive_seconds,
|
||||
@@ -348,6 +354,9 @@ class ReviewedApplicationMqttTransport:
|
||||
self._latest_device_fault = False
|
||||
self._latest_system_error_code: int | None = None
|
||||
self._latest_system_error_state: str | None = None
|
||||
self._last_loop_result_code: int | None = None
|
||||
self._last_loop_result_name: str | None = None
|
||||
self._last_loop_phase: str | None = None
|
||||
|
||||
def open(self) -> ApplicationMqttTransportSnapshot:
|
||||
with self._lock:
|
||||
@@ -562,7 +571,9 @@ class ReviewedApplicationMqttTransport:
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
self._fail_after_publish("control MQTT network loop failed", exc)
|
||||
if result != mqtt.MQTT_ERR_SUCCESS:
|
||||
self._fail_after_publish("control MQTT network loop returned an error")
|
||||
self._fail_after_publish(
|
||||
self._record_loop_failure(result, phase="maintain-open")
|
||||
)
|
||||
self._discard_allowed_responses(allowed)
|
||||
|
||||
def close(self) -> None:
|
||||
@@ -649,6 +660,9 @@ class ReviewedApplicationMqttTransport:
|
||||
latest_device_init_ready=self._latest_device_init_ready,
|
||||
latest_system_error_code=self._latest_system_error_code,
|
||||
latest_system_error_state=self._latest_system_error_state,
|
||||
last_loop_result_code=self._last_loop_result_code,
|
||||
last_loop_result_name=self._last_loop_result_name,
|
||||
last_loop_phase=self._last_loop_phase,
|
||||
operation_keys_consumed=len(self._consumed_operation_keys),
|
||||
)
|
||||
|
||||
@@ -879,9 +893,13 @@ class ReviewedApplicationMqttTransport:
|
||||
self._fail_after_publish("control MQTT network loop failed", exc)
|
||||
self._fail_before_publish("control MQTT network loop failed", exc)
|
||||
if result != mqtt.MQTT_ERR_SUCCESS:
|
||||
failure = self._record_loop_failure(
|
||||
result,
|
||||
phase="post-publish" if post_publish else "connect-subscribe",
|
||||
)
|
||||
if post_publish:
|
||||
self._fail_after_publish("control MQTT network loop returned an error")
|
||||
self._fail_before_publish("control MQTT network loop returned an error")
|
||||
self._fail_after_publish(failure)
|
||||
self._fail_before_publish(failure)
|
||||
|
||||
def _service_once(self, *, post_publish: bool) -> None:
|
||||
with self._lock:
|
||||
@@ -898,9 +916,33 @@ class ReviewedApplicationMqttTransport:
|
||||
self._fail_after_publish("control MQTT network loop failed", exc)
|
||||
self._fail_before_publish("control MQTT network loop failed", exc)
|
||||
if result != mqtt.MQTT_ERR_SUCCESS:
|
||||
failure = self._record_loop_failure(
|
||||
result,
|
||||
phase="post-publish-drain" if post_publish else "pre-publish-drain",
|
||||
)
|
||||
if post_publish:
|
||||
self._fail_after_publish("control MQTT network loop returned an error")
|
||||
self._fail_before_publish("control MQTT network loop returned an error")
|
||||
self._fail_after_publish(failure)
|
||||
self._fail_before_publish(failure)
|
||||
|
||||
def _record_loop_failure(
|
||||
self,
|
||||
result: int,
|
||||
*,
|
||||
phase: str,
|
||||
) -> str:
|
||||
result_code = int(result)
|
||||
try:
|
||||
result_name = mqtt.error_string(result_code)
|
||||
except (TypeError, ValueError):
|
||||
result_name = "unknown MQTT error"
|
||||
with self._lock:
|
||||
self._last_loop_result_code = result_code
|
||||
self._last_loop_result_name = result_name
|
||||
self._last_loop_phase = phase
|
||||
return (
|
||||
"control MQTT network loop returned an error "
|
||||
f"(phase={phase}, result={result_code}: {result_name})"
|
||||
)
|
||||
|
||||
def _drain_responses(
|
||||
self,
|
||||
|
||||
@@ -62,6 +62,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
TransportFactory = Callable[[str], ReviewedApplicationMqttTransport]
|
||||
ScanningObserver = Callable[[], None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -97,10 +98,13 @@ class InteractiveApplicationControlSession:
|
||||
*,
|
||||
transport_factory: TransportFactory = ReviewedApplicationMqttTransport,
|
||||
epoch_seconds: Callable[[], int] = lambda: int(time.time()),
|
||||
scanning_observer: ScanningObserver | None = None,
|
||||
) -> None:
|
||||
self._authority_loader = authority_loader
|
||||
self._transport_factory = transport_factory
|
||||
self._epoch_seconds = epoch_seconds
|
||||
self._scanning_observer = scanning_observer
|
||||
self._scanning_observer_errors = 0
|
||||
self._lock = threading.RLock()
|
||||
self._phase: ApplicationControlPhase = "idle"
|
||||
self._host: str | None = None
|
||||
@@ -215,6 +219,55 @@ class InteractiveApplicationControlSession:
|
||||
transport.close()
|
||||
return self.snapshot()
|
||||
|
||||
def retire_for_network_change(self) -> dict[str, object]:
|
||||
"""Retire local control ownership for one new explicit network action.
|
||||
|
||||
A correlated STOP followed by a network-loop loss in SCAN_STOPPING is
|
||||
not enough evidence to authorize another START. It is enough to retire
|
||||
the dead local socket when the operator explicitly chooses a new
|
||||
network route. Pre-START failures already classified safe for a fresh
|
||||
click are admissible here for the same reason: no modeling command was
|
||||
attempted.
|
||||
"""
|
||||
|
||||
with self._lock:
|
||||
failure = self._failure
|
||||
failed_network_change_admissible = (
|
||||
self._phase == "failed"
|
||||
and failure is not None
|
||||
and (
|
||||
failure.get("safe_to_retry") is True
|
||||
or failure.get("network_change_admissible") is True
|
||||
)
|
||||
)
|
||||
if self._phase not in {"idle", "completed", "closed"} and not (
|
||||
failed_network_change_admissible
|
||||
):
|
||||
raise ApplicationAcceptanceError(
|
||||
"control session cannot be retired for a network change"
|
||||
)
|
||||
if not self._worker_retired_locked():
|
||||
raise ApplicationAcceptanceError(
|
||||
"control session worker is still retiring"
|
||||
)
|
||||
previous_phase = self._phase
|
||||
previous_reason = (
|
||||
self._json_string(failure.get("reason_code"))
|
||||
if failure is not None
|
||||
else None
|
||||
)
|
||||
self._reset_locked()
|
||||
if previous_phase != "idle":
|
||||
logger.info(
|
||||
"K1 local control session retired for an explicit network change",
|
||||
extra={
|
||||
"event_code": "k1_control_session_retired_for_network_change",
|
||||
"reason_code": previous_reason,
|
||||
"automatic_retry": False,
|
||||
},
|
||||
)
|
||||
return self.snapshot()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Stop the process-owned socket without ever inventing a device STOP."""
|
||||
|
||||
@@ -247,6 +300,7 @@ class InteractiveApplicationControlSession:
|
||||
"scripted_transitions": False,
|
||||
"automatic_retry": False,
|
||||
"outcome_unknown": self._outcome_unknown,
|
||||
"scanning_observer_errors": self._scanning_observer_errors,
|
||||
"failure": dict(self._failure) if self._failure is not None else None,
|
||||
"dialogue": (
|
||||
dict(self._dialogue_snapshot) if self._dialogue_snapshot is not None else None
|
||||
@@ -310,6 +364,7 @@ class InteractiveApplicationControlSession:
|
||||
checkpoint=start_checkpoint,
|
||||
)
|
||||
self._set_phase("scanning")
|
||||
self._notify_scanning_observer()
|
||||
|
||||
executor.maintain_active_until_stop_requested(self._stop_requested.is_set)
|
||||
stop_confirmation = self._stop_request()
|
||||
@@ -412,11 +467,53 @@ class InteractiveApplicationControlSession:
|
||||
)
|
||||
== 1
|
||||
)
|
||||
safe_to_retry = not outcome_unknown and (
|
||||
status_reconciled_prestart_failure = (
|
||||
outcome_unknown
|
||||
and modeling_command_attempted is False
|
||||
and transport_snapshot_available
|
||||
and not diagnostic_evidence_unavailable
|
||||
and self._json_int_or_none(
|
||||
transport_snapshot.get("device_status_reports")
|
||||
)
|
||||
not in {None, 0}
|
||||
and self._json_string(
|
||||
transport_snapshot.get("latest_device_session_state")
|
||||
)
|
||||
== "ready"
|
||||
and transport_snapshot.get("latest_device_project_bound") is False
|
||||
and self._json_int_or_none(
|
||||
transport_snapshot.get("latest_system_error_code")
|
||||
)
|
||||
is None
|
||||
)
|
||||
stop_acknowledged_network_change = (
|
||||
outcome_unknown
|
||||
and failure_reason_code == "mqtt_network_loop_failed"
|
||||
and failed_phase == "awaiting-standby-confirmation"
|
||||
and dialogue_snapshot_available
|
||||
and transport_snapshot_available
|
||||
and dialogue_snapshot.get("stop_attempted") is True
|
||||
and dialogue_snapshot.get("stop_complete") is True
|
||||
and dialogue_stage == "stop-acknowledged"
|
||||
and modeling_command_attempted is True
|
||||
and self._json_string(
|
||||
transport_snapshot.get("latest_device_session_state")
|
||||
)
|
||||
== "scan_stopping"
|
||||
and transport_snapshot.get("latest_device_project_bound") is True
|
||||
and self._json_int_or_none(
|
||||
transport_snapshot.get("latest_system_error_code")
|
||||
)
|
||||
is None
|
||||
)
|
||||
safe_to_retry = status_reconciled_prestart_failure or (
|
||||
not outcome_unknown
|
||||
and (
|
||||
not transport_created
|
||||
or (transport_snapshot_available and publish_attempts == 0)
|
||||
or correlated_read_only_profile_mismatch
|
||||
)
|
||||
)
|
||||
self._failure = {
|
||||
"code": type(exc).__name__,
|
||||
"reason_code": failure_reason_code,
|
||||
@@ -451,6 +548,37 @@ class InteractiveApplicationControlSession:
|
||||
else None
|
||||
),
|
||||
"safe_to_retry": safe_to_retry,
|
||||
**(
|
||||
{
|
||||
"status_reconciliation": {
|
||||
"device_session_state": "ready",
|
||||
"device_project_bound": False,
|
||||
"system_error_code": None,
|
||||
"decision": "safe-explicit-prestart-retry",
|
||||
"automatic_retry": False,
|
||||
}
|
||||
}
|
||||
if status_reconciled_prestart_failure
|
||||
else {}
|
||||
),
|
||||
**(
|
||||
{
|
||||
"network_change_admissible": True,
|
||||
"network_change_reconciliation": {
|
||||
"device_session_state": "scan_stopping",
|
||||
"device_project_bound": True,
|
||||
"system_error_code": None,
|
||||
"stop_complete": True,
|
||||
"standby_confirmed": False,
|
||||
"decision": (
|
||||
"explicit-network-change-only-after-acknowledged-stop"
|
||||
),
|
||||
"automatic_retry": False,
|
||||
},
|
||||
}
|
||||
if stop_acknowledged_network_change
|
||||
else {}
|
||||
),
|
||||
}
|
||||
self._outcome_unknown = outcome_unknown
|
||||
self._dialogue_snapshot = dialogue_snapshot or None
|
||||
@@ -462,7 +590,8 @@ class InteractiveApplicationControlSession:
|
||||
"publish_attempts=%s modeling_command_attempted=%s "
|
||||
"diagnostic_snapshot_unavailable=%s "
|
||||
"diagnostic_evidence_unavailable=%s outcome_unknown=%s "
|
||||
"safe_to_retry=%s",
|
||||
"safe_to_retry=%s status_reconciliation=%s "
|
||||
"network_change_reconciliation=%s",
|
||||
type(exc).__name__,
|
||||
failure_reason_code,
|
||||
failed_phase,
|
||||
@@ -474,6 +603,51 @@ class InteractiveApplicationControlSession:
|
||||
diagnostic_evidence_unavailable,
|
||||
outcome_unknown,
|
||||
safe_to_retry,
|
||||
(
|
||||
"safe-explicit-prestart-retry"
|
||||
if status_reconciled_prestart_failure
|
||||
else None
|
||||
),
|
||||
(
|
||||
"explicit-network-change-only-after-acknowledged-stop"
|
||||
if stop_acknowledged_network_change
|
||||
else None
|
||||
),
|
||||
extra={
|
||||
"event_code": "k1_application_control_session_failed",
|
||||
"reason_code": failure_reason_code,
|
||||
"failed_phase": failed_phase,
|
||||
"dialogue_stage": dialogue_stage,
|
||||
"transport_state": self._json_string(
|
||||
transport_snapshot.get("state")
|
||||
),
|
||||
"publish_attempts": publish_attempts,
|
||||
"modeling_command_attempted": modeling_command_attempted,
|
||||
"outcome_unknown": outcome_unknown,
|
||||
"safe_to_retry": safe_to_retry,
|
||||
"status_reconciliation": (
|
||||
"safe-explicit-prestart-retry"
|
||||
if status_reconciled_prestart_failure
|
||||
else None
|
||||
),
|
||||
"network_change_admissible": (
|
||||
stop_acknowledged_network_change
|
||||
),
|
||||
"network_change_reconciliation": (
|
||||
"explicit-network-change-only-after-acknowledged-stop"
|
||||
if stop_acknowledged_network_change
|
||||
else None
|
||||
),
|
||||
"mqtt_loop_result_code": self._json_int_or_none(
|
||||
transport_snapshot.get("last_loop_result_code")
|
||||
),
|
||||
"mqtt_loop_result_name": self._json_string(
|
||||
transport_snapshot.get("last_loop_result_name")
|
||||
),
|
||||
"mqtt_loop_phase": self._json_string(
|
||||
transport_snapshot.get("last_loop_phase")
|
||||
),
|
||||
},
|
||||
)
|
||||
finally:
|
||||
final_dialogue_snapshot: dict[str, object] | None = None
|
||||
@@ -552,6 +726,23 @@ class InteractiveApplicationControlSession:
|
||||
with self._lock:
|
||||
self._set_phase_locked(phase)
|
||||
|
||||
def _notify_scanning_observer(self) -> None:
|
||||
observer = self._scanning_observer
|
||||
if observer is None:
|
||||
return
|
||||
try:
|
||||
observer()
|
||||
except Exception:
|
||||
with self._lock:
|
||||
self._scanning_observer_errors += 1
|
||||
logger.exception(
|
||||
"K1 confirmed SCANNING, but its acquisition observer failed",
|
||||
extra={
|
||||
"event_code": "k1_application_scanning_observer_failed",
|
||||
"automatic_retry": False,
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _executor_snapshot_safely(
|
||||
executor: PhysicalAcceptanceDialogueExecutor | None,
|
||||
|
||||
@@ -226,7 +226,7 @@ class VisualizationRuntime:
|
||||
raise RuntimeError("поток не завершился за отведённое время; повторите остановку")
|
||||
|
||||
def close(self, *, wait_seconds: float = 5.0) -> None:
|
||||
"""Stop the active source and release the process-wide visual bridge."""
|
||||
"""Stop the active source and release its acquisition-scoped bridge."""
|
||||
with self._lock:
|
||||
self._closed = True
|
||||
thread = self._thread
|
||||
@@ -429,13 +429,11 @@ class VisualizationRuntime:
|
||||
bridge: RerunBridge | None = None
|
||||
try:
|
||||
with self._lock:
|
||||
bridge = self._bridge
|
||||
if self._closed:
|
||||
publisher_aborted.set()
|
||||
if publisher_aborted.is_set():
|
||||
publisher_ready.set()
|
||||
return
|
||||
if bridge is None:
|
||||
candidate = self._bridge_factory(
|
||||
grpc_port=self._grpc_port,
|
||||
metrics=self._metrics,
|
||||
@@ -446,6 +444,9 @@ class VisualizationRuntime:
|
||||
if self._closed:
|
||||
publisher_aborted.set()
|
||||
else:
|
||||
# Every acquisition owns a fresh Rerun recording/store.
|
||||
# Reusing a RecordingStream across independent scans can
|
||||
# leave a later WebViewer without a StoreInfo event.
|
||||
self._bridge = candidate
|
||||
if publisher_aborted.is_set():
|
||||
publisher_ready.set()
|
||||
@@ -520,19 +521,17 @@ class VisualizationRuntime:
|
||||
finally:
|
||||
if bridge is not None:
|
||||
with self._lock:
|
||||
close_bridge = self._closed and (
|
||||
self._bridge is bridge or publisher_aborted.is_set()
|
||||
)
|
||||
if close_bridge and self._bridge is bridge:
|
||||
if self._bridge is bridge:
|
||||
self._bridge = None
|
||||
self._rerun_grpc_url = None
|
||||
if close_bridge:
|
||||
try:
|
||||
bridge.close()
|
||||
except BaseException as exc:
|
||||
# Process shutdown must not become a false successful
|
||||
# idle state when the native bridge failed to close.
|
||||
# A failed native disconnect must not become a false
|
||||
# successful idle state or leak port 9876 into the next
|
||||
# independent acquisition.
|
||||
publisher_error.append(exc)
|
||||
self._notify()
|
||||
|
||||
def join_publisher() -> None:
|
||||
# Never orphan a publisher: the session thread remains its owner.
|
||||
@@ -596,6 +595,7 @@ class VisualizationRuntime:
|
||||
self._message = message
|
||||
self._foxglove_ws_url = None
|
||||
self._foxglove_viewer_url = None
|
||||
self._rerun_grpc_url = None
|
||||
self._notify()
|
||||
|
||||
def _finish_error(self, message: str) -> None:
|
||||
@@ -605,6 +605,7 @@ class VisualizationRuntime:
|
||||
self._message = message
|
||||
self._foxglove_ws_url = None
|
||||
self._foxglove_viewer_url = None
|
||||
self._rerun_grpc_url = None
|
||||
self._notify()
|
||||
|
||||
def _current_scene_settings(self) -> RerunSceneSettings:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import socket
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import suppress
|
||||
@@ -31,6 +33,7 @@ TRAJECTORY_MIN_DISTANCE_METERS = 0.02
|
||||
TRAJECTORY_PUBLISH_INTERVAL_NS = 500_000_000
|
||||
LIVE_GRPC_BUFFER_LIMIT = "32MiB"
|
||||
DEFAULT_GRPC_PORT = 9876
|
||||
GRPC_PORT_SEARCH_SPAN = 128
|
||||
DEFAULT_CORS_ORIGINS = (
|
||||
"http://127.0.0.1:5173",
|
||||
"http://localhost:5173",
|
||||
@@ -40,6 +43,8 @@ DEFAULT_CORS_ORIGINS = (
|
||||
"http://localhost:8000",
|
||||
)
|
||||
|
||||
logger = logging.getLogger("k1link.device_plugins.xgrids_k1.viewer_receiver")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RerunSceneSettings:
|
||||
@@ -62,6 +67,34 @@ class RerunSceneSettings:
|
||||
SettingsProvider = Callable[[], RerunSceneSettings]
|
||||
|
||||
|
||||
def _select_available_grpc_port(
|
||||
preferred_port: int,
|
||||
*,
|
||||
search_span: int = GRPC_PORT_SEARCH_SPAN,
|
||||
) -> int:
|
||||
"""Select a local Rerun port without reusing a still-served recording."""
|
||||
|
||||
if not 1 <= preferred_port <= 65_535:
|
||||
raise ValueError("Rerun gRPC port must be between 1 and 65535")
|
||||
if search_span < 1:
|
||||
raise ValueError("Rerun gRPC port search span must be positive")
|
||||
|
||||
last_port = min(preferred_port + search_span, 65_536)
|
||||
for candidate in range(preferred_port, last_port):
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
||||
try:
|
||||
# Rerun binds all local interfaces. Probe the same address class
|
||||
# so a previous recording retained by a late viewer is detected.
|
||||
probe.bind(("0.0.0.0", candidate))
|
||||
except OSError:
|
||||
continue
|
||||
return candidate
|
||||
raise RuntimeError(
|
||||
"No local Rerun gRPC port is available in "
|
||||
f"{preferred_port}..{last_port - 1}"
|
||||
)
|
||||
|
||||
|
||||
class RerunBridge:
|
||||
"""Publish transport-neutral canonical envelopes to a Rerun recording."""
|
||||
|
||||
@@ -85,9 +118,20 @@ class RerunBridge:
|
||||
else:
|
||||
recording = recording_factory("nodedc_mission_core_spatial")
|
||||
try:
|
||||
selected_grpc_port = _select_available_grpc_port(grpc_port)
|
||||
if selected_grpc_port != grpc_port:
|
||||
logger.info(
|
||||
"Mission Core selected a new Rerun port because an earlier "
|
||||
"viewer still owns the preferred listener",
|
||||
extra={
|
||||
"event_code": "rerun_grpc_port_rotated",
|
||||
"preferred_port": grpc_port,
|
||||
"selected_port": selected_grpc_port,
|
||||
},
|
||||
)
|
||||
blueprint = _blueprint(self._settings)
|
||||
url = recording.serve_grpc(
|
||||
grpc_port=grpc_port,
|
||||
grpc_port=selected_grpc_port,
|
||||
default_blueprint=blueprint,
|
||||
# This is a reconnect cushion for the live preview, not the source
|
||||
# of record. Raw MQTT evidence is persisted independently. A large
|
||||
@@ -132,7 +176,7 @@ class RerunBridge:
|
||||
return self._url
|
||||
|
||||
def begin_session(self, metrics: BridgeMetrics | None = None) -> None:
|
||||
"""Reset session state while keeping the process-lifetime server alive."""
|
||||
"""Initialize the one acquisition owned by this recording server."""
|
||||
if self._closed:
|
||||
raise RuntimeError("Rerun bridge is already closed")
|
||||
if metrics is not None:
|
||||
|
||||
@@ -53,8 +53,10 @@ from k1link.web.plugin_runtime import (
|
||||
PluginRuntimeUnavailableError,
|
||||
)
|
||||
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
|
||||
from k1link.web.runtime_diagnostics import configure_scanner_diagnostics
|
||||
from k1link.web.session_api import build_session_router
|
||||
from k1link.web.system_telemetry_api import build_system_telemetry_router
|
||||
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
||||
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
|
||||
@@ -274,6 +276,9 @@ async def _recording_preparation_reconciler() -> None:
|
||||
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
reconciler: asyncio.Task[None] | None = None
|
||||
try:
|
||||
configure_scanner_diagnostics(
|
||||
REPOSITORY_ROOT / ".runtime" / "mission-core" / "logs"
|
||||
)
|
||||
session_recording_preparation_manager.start()
|
||||
# Recovery is intentionally a one-shot startup phase. The archive
|
||||
# helper owns a cross-process lease, while ordinary catalog requests
|
||||
@@ -671,6 +676,7 @@ app.include_router(
|
||||
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
||||
)
|
||||
)
|
||||
app.include_router(build_viewer_diagnostics_router())
|
||||
|
||||
|
||||
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
SCANNER_LOGGER_NAME: Final = "k1link.device_plugins.xgrids_k1"
|
||||
SCANNER_DIAGNOSTIC_FILE: Final = "scanner-diagnostics.jsonl"
|
||||
_HANDLER_MARKER: Final = "_mission_core_scanner_diagnostics_path"
|
||||
_EXTRA_FIELDS: Final = (
|
||||
"event_code",
|
||||
"reason_code",
|
||||
"failed_phase",
|
||||
"dialogue_stage",
|
||||
"transport_state",
|
||||
"publish_attempts",
|
||||
"modeling_command_attempted",
|
||||
"outcome_unknown",
|
||||
"safe_to_retry",
|
||||
"status_reconciliation",
|
||||
"network_change_admissible",
|
||||
"network_change_reconciliation",
|
||||
"mqtt_loop_result_code",
|
||||
"mqtt_loop_result_name",
|
||||
"mqtt_loop_phase",
|
||||
"automatic_retry",
|
||||
"camera_source_id",
|
||||
"evidence_session_id",
|
||||
"activation_trigger",
|
||||
"lease_generation",
|
||||
"lease_state",
|
||||
"recovery_strategy",
|
||||
"endpoint_reachable",
|
||||
"host_route_class",
|
||||
"address_changed",
|
||||
"device_write_performed",
|
||||
"failure_stage",
|
||||
"start_checkpoint_released",
|
||||
"stale_session_retired",
|
||||
"stream_id",
|
||||
"backend_activity_sequence",
|
||||
"viewer_range_max_ns",
|
||||
"stalled_for_ms",
|
||||
"recovery_attempt",
|
||||
"preferred_port",
|
||||
"selected_port",
|
||||
)
|
||||
|
||||
|
||||
class ScannerDiagnosticJsonFormatter(logging.Formatter):
|
||||
"""Serialize a bounded, secret-free scanner diagnostic record."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
document: dict[str, object] = {
|
||||
"timestamp_utc": datetime.fromtimestamp(record.created, UTC)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": record.getMessage()[:1000],
|
||||
}
|
||||
for field in _EXTRA_FIELDS:
|
||||
value = getattr(record, field, None)
|
||||
if value is not None and isinstance(value, (str, int, bool)):
|
||||
document[field] = value
|
||||
return json.dumps(document, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def configure_scanner_diagnostics(logs_root: Path) -> Path:
|
||||
"""Install one private rotating JSONL handler for K1 field diagnostics."""
|
||||
|
||||
logs_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
os.chmod(logs_root, 0o700)
|
||||
target = (logs_root / SCANNER_DIAGNOSTIC_FILE).resolve()
|
||||
logger = logging.getLogger(SCANNER_LOGGER_NAME)
|
||||
for handler in logger.handlers:
|
||||
if getattr(handler, _HANDLER_MARKER, None) == str(target):
|
||||
return target
|
||||
|
||||
handler = RotatingFileHandler(
|
||||
target,
|
||||
maxBytes=5 * 1024 * 1024,
|
||||
backupCount=5,
|
||||
encoding="utf-8",
|
||||
delay=False,
|
||||
)
|
||||
os.chmod(target, 0o600)
|
||||
handler.setLevel(logging.INFO)
|
||||
handler.setFormatter(ScannerDiagnosticJsonFormatter())
|
||||
setattr(handler, _HANDLER_MARKER, str(target))
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(logging.INFO)
|
||||
return target
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Response
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
logger = logging.getLogger("k1link.device_plugins.xgrids_k1.viewer_receiver")
|
||||
|
||||
LiveViewerEventCode = Literal[
|
||||
"live_receiver_stalled",
|
||||
"live_receiver_restart_requested",
|
||||
"live_receiver_recovered",
|
||||
"live_receiver_recovery_exhausted",
|
||||
"live_receiver_active_store_admitted",
|
||||
"live_receiver_error",
|
||||
]
|
||||
LiveViewerFailureStage = Literal[
|
||||
"recording-open-timeout",
|
||||
"viewer-start",
|
||||
"module-load",
|
||||
"receiver-stalled",
|
||||
]
|
||||
|
||||
|
||||
class LiveViewerDiagnosticEvent(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
schema_version: Literal["missioncore.live-viewer-diagnostic/v1"]
|
||||
event_code: LiveViewerEventCode
|
||||
failure_stage: LiveViewerFailureStage | None = None
|
||||
stream_id: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$",
|
||||
)
|
||||
backend_activity_sequence: int | None = Field(default=None, ge=0)
|
||||
viewer_range_max_ns: int | None = Field(default=None, ge=0)
|
||||
stalled_for_ms: int | None = Field(default=None, ge=0, le=600_000)
|
||||
recovery_attempt: int | None = Field(default=None, ge=1, le=3)
|
||||
|
||||
|
||||
def build_viewer_diagnostics_router() -> APIRouter:
|
||||
router = APIRouter(prefix="/api/v1/viewer", tags=["viewer"])
|
||||
|
||||
@router.post("/live-diagnostics", status_code=204)
|
||||
def record_live_diagnostic(event: LiveViewerDiagnosticEvent) -> Response:
|
||||
logger.info(
|
||||
"Mission Core live Rerun receiver diagnostic: event=%s stream=%s",
|
||||
event.event_code,
|
||||
event.stream_id,
|
||||
extra={
|
||||
"event_code": event.event_code,
|
||||
"failure_stage": event.failure_stage,
|
||||
"stream_id": event.stream_id,
|
||||
"backend_activity_sequence": event.backend_activity_sequence,
|
||||
"viewer_range_max_ns": event.viewer_range_max_ns,
|
||||
"stalled_for_ms": event.stalled_for_ms,
|
||||
"recovery_attempt": event.recovery_attempt,
|
||||
},
|
||||
)
|
||||
return Response(status_code=204)
|
||||
|
||||
return router
|
||||
@@ -136,6 +136,31 @@ def test_live_ingress_allows_only_one_worker_consumer() -> None:
|
||||
ingress.open_consumer("worker-2")
|
||||
|
||||
|
||||
def test_live_ingress_new_session_discards_queued_events_from_previous_session() -> None:
|
||||
ingress = LivePerceptionIngress()
|
||||
ingress.open_consumer("worker-1")
|
||||
ingress.begin_session("session-1")
|
||||
assert ingress.publish(
|
||||
modality="lidar",
|
||||
source_id="lixel/application/report/lio_pcl",
|
||||
source_sequence=1,
|
||||
captured_at_epoch_ns=1,
|
||||
received_monotonic_ns=1,
|
||||
payload=b"old-session",
|
||||
)
|
||||
|
||||
ingress.end_session("session-1")
|
||||
ingress.begin_session("session-2")
|
||||
|
||||
events = []
|
||||
while (event := ingress.take_next("worker-1", timeout=0)) is not None:
|
||||
events.append(event)
|
||||
assert len(events) == 1
|
||||
assert events[0].session_id == "session-2"
|
||||
assert events[0].modality == "control"
|
||||
assert events[0].payload == b'{"event":"session-start"}'
|
||||
|
||||
|
||||
def test_live_result_round_trip_keeps_video_mask_boxes_and_shadow_authority() -> None:
|
||||
mask = np.zeros((600, 800), dtype=np.uint8)
|
||||
mask[100:120, 200:240] = 4
|
||||
|
||||
+40
-16
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import socket
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
@@ -20,6 +22,7 @@ from k1link.viewer.rerun_bridge import (
|
||||
RerunSceneSettings,
|
||||
_live_time_panel,
|
||||
_point_colors,
|
||||
_select_available_grpc_port,
|
||||
)
|
||||
|
||||
|
||||
@@ -62,6 +65,33 @@ class DisconnectFailureRecording(FakeRecording):
|
||||
raise RuntimeError("synthetic disconnect failure")
|
||||
|
||||
|
||||
def test_rerun_port_selection_skips_a_recording_still_held_by_a_viewer(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as occupied:
|
||||
occupied.bind(("0.0.0.0", 0))
|
||||
occupied.listen()
|
||||
preferred_port = int(occupied.getsockname()[1])
|
||||
|
||||
selected_port = _select_available_grpc_port(preferred_port, search_span=8)
|
||||
recording = FakeRecording()
|
||||
with caplog.at_level(
|
||||
logging.INFO,
|
||||
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
||||
):
|
||||
bridge = RerunBridge(
|
||||
grpc_port=preferred_port,
|
||||
recording_factory=lambda _: recording, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert selected_port != preferred_port
|
||||
assert preferred_port < selected_port < preferred_port + 8
|
||||
assert caplog.records[-1].event_code == "rerun_grpc_port_rotated"
|
||||
assert caplog.records[-1].preferred_port == preferred_port
|
||||
assert caplog.records[-1].selected_port == selected_port
|
||||
bridge.close()
|
||||
|
||||
|
||||
def _message(
|
||||
topic: str,
|
||||
payload: bytes,
|
||||
@@ -266,7 +296,7 @@ def test_palettes_are_deterministic_and_custom_color_is_exact() -> None:
|
||||
assert custom_over_rgb.tolist() == [[16, 32, 48], [16, 32, 48]]
|
||||
|
||||
|
||||
def test_runtime_reuses_one_bridge_across_sequential_sessions(tmp_path: Path) -> None:
|
||||
def test_runtime_owns_fresh_bridge_for_each_sequential_session(tmp_path: Path) -> None:
|
||||
capture = tmp_path / "mqtt.raw.k1mqtt"
|
||||
point_topic = "RealtimePointcloud"
|
||||
pose_topic = "RealtimePath"
|
||||
@@ -300,10 +330,12 @@ def test_runtime_reuses_one_bridge_across_sequential_sessions(tmp_path: Path) ->
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
recording = FakeRecording()
|
||||
recordings: list[FakeRecording] = []
|
||||
created: list[RerunBridge] = []
|
||||
|
||||
def bridge_factory(**kwargs: object) -> RerunBridge:
|
||||
recording = FakeRecording()
|
||||
recordings.append(recording)
|
||||
bridge = RerunBridge(
|
||||
recording_factory=lambda _: recording, # type: ignore[arg-type]
|
||||
**kwargs, # type: ignore[arg-type]
|
||||
@@ -329,8 +361,8 @@ def test_runtime_reuses_one_bridge_across_sequential_sessions(tmp_path: Path) ->
|
||||
assert snapshot["metrics"]["mqtt_to_publish_ms"] is None
|
||||
runtime.stop(wait_seconds=5.0)
|
||||
assert runtime.snapshot()["phase"] == "idle"
|
||||
assert runtime.snapshot()["rerun_grpc_url"] == "rerun+http://127.0.0.1:9876/proxy"
|
||||
assert recording.disconnected is False
|
||||
assert runtime.snapshot()["rerun_grpc_url"] is None
|
||||
assert recordings[0].disconnected is True
|
||||
|
||||
runtime.start_replay(capture, speed=0.0)
|
||||
deadline = time.monotonic() + 5.0
|
||||
@@ -340,13 +372,12 @@ def test_runtime_reuses_one_bridge_across_sequential_sessions(tmp_path: Path) ->
|
||||
time.sleep(0.05)
|
||||
runtime.stop(wait_seconds=5.0)
|
||||
|
||||
assert len(created) == 1
|
||||
assert len(created) == 2
|
||||
assert runtime.snapshot()["metrics"]["pcl_frames"] == 1
|
||||
assert runtime.snapshot()["rerun_grpc_url"] == "rerun+http://127.0.0.1:9876/proxy"
|
||||
assert recording.disconnected is False
|
||||
assert runtime.snapshot()["rerun_grpc_url"] is None
|
||||
assert all(recording.disconnected for recording in recordings)
|
||||
runtime.close()
|
||||
assert runtime.snapshot()["rerun_grpc_url"] is None
|
||||
assert recording.disconnected is True
|
||||
|
||||
|
||||
def test_runtime_reports_bridge_close_failure_instead_of_false_idle(tmp_path: Path) -> None:
|
||||
@@ -385,18 +416,11 @@ def test_runtime_reports_bridge_close_failure_instead_of_false_idle(tmp_path: Pa
|
||||
time.sleep(0.01)
|
||||
snapshot = runtime.snapshot()
|
||||
|
||||
assert snapshot["phase"] == "idle"
|
||||
assert snapshot["rerun_grpc_url"] == "rerun+http://127.0.0.1:9876/proxy"
|
||||
assert recording.disconnected is False
|
||||
|
||||
with pytest.raises(RuntimeError, match="synthetic disconnect failure"):
|
||||
runtime.close()
|
||||
|
||||
snapshot = runtime.snapshot()
|
||||
assert snapshot["phase"] == "error"
|
||||
assert "synthetic disconnect failure" in snapshot["message"]
|
||||
assert snapshot["rerun_grpc_url"] is None
|
||||
assert recording.disconnected is True
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_close_during_blocked_factory_closes_the_late_bridge(tmp_path: Path) -> None:
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import stat
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import APIRouter
|
||||
from fastapi.routing import APIRoute
|
||||
from pydantic import ValidationError
|
||||
|
||||
from k1link.web.runtime_diagnostics import (
|
||||
SCANNER_LOGGER_NAME,
|
||||
configure_scanner_diagnostics,
|
||||
)
|
||||
from k1link.web.viewer_diagnostics_api import (
|
||||
LiveViewerDiagnosticEvent,
|
||||
build_viewer_diagnostics_router,
|
||||
)
|
||||
|
||||
|
||||
def _endpoint(router: APIRouter, path: str, method: str) -> Callable[..., Any]:
|
||||
for route in router.routes:
|
||||
if (
|
||||
isinstance(route, APIRoute)
|
||||
and route.path == path
|
||||
and route.methods is not None
|
||||
and method in route.methods
|
||||
):
|
||||
return route.endpoint
|
||||
raise AssertionError(f"{method} {path} route is missing")
|
||||
|
||||
|
||||
def test_private_scanner_diagnostics_are_durable_structured_and_bounded(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
target = configure_scanner_diagnostics(tmp_path / "logs")
|
||||
logger = logging.getLogger(f"{SCANNER_LOGGER_NAME}.test")
|
||||
logger.error(
|
||||
"field control failure",
|
||||
extra={
|
||||
"event_code": "k1_application_control_session_failed",
|
||||
"reason_code": "mqtt_network_loop_failed",
|
||||
"mqtt_loop_result_code": 7,
|
||||
"mqtt_loop_result_name": "The connection was lost.",
|
||||
"mqtt_loop_phase": "post-publish-drain",
|
||||
"automatic_retry": False,
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"evidence_session_id": "20260728T163450Z_viewer_live",
|
||||
"activation_trigger": "application-control-scanning",
|
||||
"network_change_admissible": True,
|
||||
"network_change_reconciliation": (
|
||||
"explicit-network-change-only-after-acknowledged-stop"
|
||||
),
|
||||
"lease_generation": 3,
|
||||
"lease_state": "reachable",
|
||||
"recovery_strategy": "existing-mqtt-endpoint",
|
||||
"endpoint_reachable": True,
|
||||
"address_changed": False,
|
||||
"device_write_performed": False,
|
||||
"preferred_port": 9876,
|
||||
"selected_port": 9877,
|
||||
"unapproved_secret_field": "must-not-be-written",
|
||||
},
|
||||
)
|
||||
for handler in logging.getLogger(SCANNER_LOGGER_NAME).handlers:
|
||||
handler.flush()
|
||||
|
||||
document = json.loads(target.read_text(encoding="utf-8").splitlines()[-1])
|
||||
assert stat.S_IMODE(target.parent.stat().st_mode) == 0o700
|
||||
assert stat.S_IMODE(target.stat().st_mode) == 0o600
|
||||
assert document["event_code"] == "k1_application_control_session_failed"
|
||||
assert document["reason_code"] == "mqtt_network_loop_failed"
|
||||
assert document["mqtt_loop_result_code"] == 7
|
||||
assert document["mqtt_loop_phase"] == "post-publish-drain"
|
||||
assert document["automatic_retry"] is False
|
||||
assert document["camera_source_id"] == "sensor.camera.right"
|
||||
assert document["evidence_session_id"] == "20260728T163450Z_viewer_live"
|
||||
assert document["activation_trigger"] == "application-control-scanning"
|
||||
assert document["network_change_admissible"] is True
|
||||
assert document["network_change_reconciliation"] == (
|
||||
"explicit-network-change-only-after-acknowledged-stop"
|
||||
)
|
||||
assert document["lease_generation"] == 3
|
||||
assert document["lease_state"] == "reachable"
|
||||
assert document["recovery_strategy"] == "existing-mqtt-endpoint"
|
||||
assert document["endpoint_reachable"] is True
|
||||
assert document["address_changed"] is False
|
||||
assert document["device_write_performed"] is False
|
||||
assert document["preferred_port"] == 9876
|
||||
assert document["selected_port"] == 9877
|
||||
assert "unapproved_secret_field" not in document
|
||||
|
||||
parent = logging.getLogger(SCANNER_LOGGER_NAME)
|
||||
for handler in list(parent.handlers):
|
||||
if getattr(handler, "baseFilename", None) == str(target):
|
||||
parent.removeHandler(handler)
|
||||
handler.close()
|
||||
|
||||
|
||||
def test_live_viewer_diagnostic_endpoint_accepts_only_bounded_events(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
router = build_viewer_diagnostics_router()
|
||||
endpoint = _endpoint(router, "/api/v1/viewer/live-diagnostics", "POST")
|
||||
event = LiveViewerDiagnosticEvent(
|
||||
schema_version="missioncore.live-viewer-diagnostic/v1",
|
||||
event_code="live_receiver_stalled",
|
||||
failure_stage="receiver-stalled",
|
||||
stream_id="acquisition-123",
|
||||
backend_activity_sequence=8_572,
|
||||
viewer_range_max_ns=231_000_000_000,
|
||||
stalled_for_ms=5_500,
|
||||
recovery_attempt=1,
|
||||
)
|
||||
|
||||
with caplog.at_level(
|
||||
logging.INFO,
|
||||
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
||||
):
|
||||
response = endpoint(event)
|
||||
|
||||
assert response.status_code == 204
|
||||
assert "event=live_receiver_stalled" in caplog.text
|
||||
assert caplog.records[-1].failure_stage == "receiver-stalled"
|
||||
with pytest.raises(ValidationError):
|
||||
LiveViewerDiagnosticEvent.model_validate(
|
||||
{
|
||||
**event.model_dump(),
|
||||
"source_url": "http://192.168.56.1/private",
|
||||
}
|
||||
)
|
||||
fallback = LiveViewerDiagnosticEvent(
|
||||
schema_version="missioncore.live-viewer-diagnostic/v1",
|
||||
event_code="live_receiver_active_store_admitted",
|
||||
stream_id="acquisition-123",
|
||||
backend_activity_sequence=8_573,
|
||||
)
|
||||
assert fallback.failure_stage is None
|
||||
@@ -128,3 +128,60 @@ def test_read_wifi_status_once_reads_only_and_returns_current_dhcp_address(
|
||||
assert result["operation"] == "single_reviewed_wifi_status_read"
|
||||
assert result["write_performed"] is False
|
||||
assert result["status"]["ipv4"] == "10.255.254.77"
|
||||
|
||||
|
||||
def test_read_wifi_status_recovery_rediscover_ignores_retained_handle(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
value = bytearray(54)
|
||||
value[0] = 11
|
||||
value[1:12] = b"WIFI_CLIENT"
|
||||
value[33] = 4
|
||||
value[34:38] = bytes((10, 255, 254, 77))
|
||||
value[50] = 1
|
||||
stale_handle = object()
|
||||
recovered_handle = object()
|
||||
characteristic = SimpleNamespace(
|
||||
uuid=wifi_module.STATUS_CHARACTERISTIC_UUID,
|
||||
service_uuid=wifi_module.SERVICE_UUID,
|
||||
properties=["read"],
|
||||
)
|
||||
service = SimpleNamespace(uuid=wifi_module.SERVICE_UUID)
|
||||
|
||||
class FakeServices:
|
||||
def get_service(self, uuid: str) -> object | None:
|
||||
return service if uuid == wifi_module.SERVICE_UUID else None
|
||||
|
||||
def get_characteristic(self, uuid: str) -> object | None:
|
||||
return characteristic if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID else None
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, device: object, **_kwargs: object) -> None:
|
||||
assert device is recovered_handle
|
||||
self.services = FakeServices()
|
||||
self.name = "XGR-K1"
|
||||
|
||||
async def __aenter__(self) -> Any:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
async def read_gatt_char(self, _characteristic: object) -> bytes:
|
||||
return bytes(value)
|
||||
|
||||
async def rediscover(*_args: object, **_kwargs: object) -> object:
|
||||
return recovered_handle
|
||||
|
||||
monkeypatch.setattr(wifi_module, "discovered_device", lambda _uuid: stale_handle)
|
||||
monkeypatch.setattr(wifi_module.BleakScanner, "find_device_by_address", rediscover)
|
||||
monkeypatch.setattr(wifi_module, "BleakClient", FakeClient)
|
||||
|
||||
result = asyncio.run(
|
||||
read_wifi_status_once(
|
||||
"synthetic-corebluetooth-uuid",
|
||||
rediscover=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert result["status"]["ipv4"] == "10.255.254.77"
|
||||
|
||||
@@ -247,6 +247,8 @@ def test_verify_connection_refreshes_dynamic_dhcp_address_and_rotates_session(
|
||||
assert state["device_session"]["device_session_id"] != "old-device-session"
|
||||
assert state["connection_verification"] == {
|
||||
"status": "live-address-observed",
|
||||
"lease_state": "configured",
|
||||
"lease_generation": 1,
|
||||
"endpoint_validation": "ble-wifi-status-read",
|
||||
"network_reachability": "not-probed",
|
||||
"address_changed": True,
|
||||
@@ -283,7 +285,7 @@ def test_implicit_acquisition_target_uses_current_ble_dhcp_address(
|
||||
assert state["acquisition"]["target_host"] == "10.255.254.77"
|
||||
|
||||
|
||||
def test_control_session_opens_against_current_ble_dhcp_address(
|
||||
def test_control_session_reuses_reachable_process_owned_connection_lease(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
@@ -291,6 +293,8 @@ def test_control_session_opens_against_current_ble_dhcp_address(
|
||||
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
||||
service._connection_mode = "bridge" # noqa: SLF001
|
||||
service._k1_ip = "10.255.254.54" # noqa: SLF001
|
||||
service._device_id = "known-k1" # noqa: SLF001
|
||||
service._device_session_id = "known-session" # noqa: SLF001
|
||||
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||
opened_hosts: list[str] = []
|
||||
|
||||
@@ -302,12 +306,12 @@ def test_control_session_opens_against_current_ble_dhcp_address(
|
||||
opened_hosts.append(host)
|
||||
return self.snapshot()
|
||||
|
||||
async def fake_status_read(*_: object, **__: object) -> dict[str, Any]:
|
||||
return _wifi_status_read("10.255.254.77")
|
||||
async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]:
|
||||
raise AssertionError("reachable connection lease must not reopen BLE")
|
||||
|
||||
service._application_control_session = FakeOpenControlSession() # type: ignore[assignment] # noqa: SLF001
|
||||
monkeypatch.setattr(facade_module, "read_wifi_status_once", fake_status_read)
|
||||
monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False)
|
||||
monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read)
|
||||
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True)
|
||||
|
||||
state = service.open_application_control_session(
|
||||
OpenApplicationControlSessionRequest(
|
||||
@@ -320,8 +324,241 @@ def test_control_session_opens_against_current_ble_dhcp_address(
|
||||
)
|
||||
)
|
||||
|
||||
assert opened_hosts == ["10.255.254.54"]
|
||||
assert state["k1_ip"] == "10.255.254.54"
|
||||
assert state["connection_verification"]["lease_state"] == "reachable"
|
||||
assert state["connection_verification"]["network_reachability"] == "reachable"
|
||||
open_operation = next(
|
||||
operation
|
||||
for operation in state["operations"]
|
||||
if operation["action"] == "application-control.session.open"
|
||||
)
|
||||
assert open_operation["status"] == "succeeded"
|
||||
assert open_operation["result"] == {
|
||||
"lease_generation": 0,
|
||||
"connection_lease_reused": True,
|
||||
"recovery_performed": False,
|
||||
"address_changed": False,
|
||||
"device_write_performed": False,
|
||||
}
|
||||
|
||||
|
||||
def test_reachable_connection_lease_supports_repeated_independent_control_sessions(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, _ = service_with_fake_runtime(tmp_path)
|
||||
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
||||
service._connection_mode = "bridge" # noqa: SLF001
|
||||
service._k1_ip = "10.255.254.54" # noqa: SLF001
|
||||
service._device_id = "known-k1" # noqa: SLF001
|
||||
service._device_session_id = "known-session" # noqa: SLF001
|
||||
service._connection_lease_generation = 7 # noqa: SLF001
|
||||
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||
opened_hosts: list[str] = []
|
||||
|
||||
class FakeCompletedControlSession:
|
||||
def snapshot(self) -> dict[str, object]:
|
||||
return {
|
||||
"state": "completed",
|
||||
"can_open": True,
|
||||
"can_confirm_standby": False,
|
||||
}
|
||||
|
||||
def open(self, *, host: str, **_: object) -> dict[str, object]:
|
||||
opened_hosts.append(host)
|
||||
return self.snapshot()
|
||||
|
||||
async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]:
|
||||
raise AssertionError("repeated scans must not repeat BLE or Wi-Fi setup")
|
||||
|
||||
service._application_control_session = FakeCompletedControlSession() # type: ignore[assignment] # noqa: SLF001
|
||||
monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read)
|
||||
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True)
|
||||
request = OpenApplicationControlSessionRequest(
|
||||
operator_present=True,
|
||||
owner_controlled_device=True,
|
||||
lixelgo_closed=True,
|
||||
battery_storage_confirmed=True,
|
||||
expected_physical_state_confirmed=True,
|
||||
timezone_name="Europe/Moscow",
|
||||
)
|
||||
|
||||
service.open_application_control_session(request)
|
||||
state = service.open_application_control_session(request)
|
||||
|
||||
assert opened_hosts == ["10.255.254.54", "10.255.254.54"]
|
||||
assert state["device_session"]["device_session_id"] == "known-session"
|
||||
assert state["connection_verification"]["lease_generation"] == 7
|
||||
open_operations = [
|
||||
operation
|
||||
for operation in state["operations"]
|
||||
if operation["action"] == "application-control.session.open"
|
||||
]
|
||||
assert len(open_operations) == 2
|
||||
assert {operation["status"] for operation in open_operations} == {"succeeded"}
|
||||
assert all(
|
||||
operation["result"]["connection_lease_reused"] is True for operation in open_operations
|
||||
)
|
||||
|
||||
|
||||
def test_control_session_recovers_changed_bridge_address_without_wifi_write(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, _ = service_with_fake_runtime(tmp_path)
|
||||
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
||||
service._connection_mode = "bridge" # noqa: SLF001
|
||||
service._k1_ip = "10.255.254.54" # noqa: SLF001
|
||||
service._device_id = "known-k1" # noqa: SLF001
|
||||
service._device_session_id = "old-session" # noqa: SLF001
|
||||
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||
opened_hosts: list[str] = []
|
||||
status_calls: list[dict[str, object]] = []
|
||||
|
||||
class FakeOpenControlSession:
|
||||
def snapshot(self) -> dict[str, object]:
|
||||
return {"state": "idle", "can_confirm_standby": False}
|
||||
|
||||
def open(self, *, host: str, **_: object) -> dict[str, object]:
|
||||
opened_hosts.append(host)
|
||||
return self.snapshot()
|
||||
|
||||
async def fake_status_read(*_: object, **kwargs: object) -> dict[str, Any]:
|
||||
status_calls.append(kwargs)
|
||||
return _wifi_status_read("10.255.254.77")
|
||||
|
||||
service._application_control_session = FakeOpenControlSession() # type: ignore[assignment] # noqa: SLF001
|
||||
monkeypatch.setattr(facade_module, "read_wifi_status_once", fake_status_read)
|
||||
monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False)
|
||||
monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed")
|
||||
monkeypatch.setattr(
|
||||
facade_module,
|
||||
"_control_endpoint_reachable",
|
||||
lambda target: target == "10.255.254.77",
|
||||
)
|
||||
|
||||
state = service.open_application_control_session(
|
||||
OpenApplicationControlSessionRequest(
|
||||
operator_present=True,
|
||||
owner_controlled_device=True,
|
||||
lixelgo_closed=True,
|
||||
battery_storage_confirmed=True,
|
||||
expected_physical_state_confirmed=True,
|
||||
timezone_name="Europe/Moscow",
|
||||
)
|
||||
)
|
||||
|
||||
assert status_calls == [{"timeout_seconds": 20.0, "rediscover": True}]
|
||||
assert opened_hosts == ["10.255.254.77"]
|
||||
assert state["k1_ip"] == "10.255.254.77"
|
||||
assert state["device_session"]["device_session_id"] != "old-session"
|
||||
assert state["connection_verification"]["status"] == "recovered"
|
||||
assert state["connection_verification"]["write_performed"] is False
|
||||
open_operation = next(
|
||||
operation
|
||||
for operation in state["operations"]
|
||||
if operation["action"] == "application-control.session.open"
|
||||
)
|
||||
assert open_operation["result"]["recovery_performed"] is True
|
||||
assert open_operation["result"]["address_changed"] is True
|
||||
assert open_operation["result"]["device_write_performed"] is False
|
||||
|
||||
|
||||
def test_control_session_prestart_failure_is_journaled_and_marks_lease_offline(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, _ = service_with_fake_runtime(tmp_path)
|
||||
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
||||
service._connection_mode = "bridge" # noqa: SLF001
|
||||
service._k1_ip = "10.255.254.54" # noqa: SLF001
|
||||
service._device_id = "known-k1" # noqa: SLF001
|
||||
service._device_session_id = "known-session" # noqa: SLF001
|
||||
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||
|
||||
async def failed_status_read(*_: object, **__: object) -> dict[str, Any]:
|
||||
raise TimeoutError("synthetic BLE recovery timeout")
|
||||
|
||||
monkeypatch.setattr(facade_module, "read_wifi_status_once", failed_status_read)
|
||||
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: False)
|
||||
monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed")
|
||||
|
||||
with pytest.raises(
|
||||
facade_module.ConnectionLeaseUnavailable,
|
||||
match="повторное чтение состояния",
|
||||
):
|
||||
service.open_application_control_session(
|
||||
OpenApplicationControlSessionRequest(
|
||||
operator_present=True,
|
||||
owner_controlled_device=True,
|
||||
lixelgo_closed=True,
|
||||
battery_storage_confirmed=True,
|
||||
expected_physical_state_confirmed=True,
|
||||
timezone_name="Europe/Moscow",
|
||||
)
|
||||
)
|
||||
|
||||
state = service.state()
|
||||
assert state["application_control_session"]["state"] == "idle"
|
||||
assert state["device_session"]["connectivity"] == "offline"
|
||||
assert state["connection_verification"]["lease_state"] == "disconnected"
|
||||
open_operation = next(
|
||||
operation
|
||||
for operation in state["operations"]
|
||||
if operation["action"] == "application-control.session.open"
|
||||
)
|
||||
assert open_operation["status"] == "failed"
|
||||
assert open_operation["error"] == {
|
||||
"category": "connection",
|
||||
"code": "connection_lease_ble_recovery_failed",
|
||||
"retryable": False,
|
||||
"safe_to_retry": True,
|
||||
"side_effect_status": "none",
|
||||
}
|
||||
|
||||
|
||||
def test_bridge_route_mismatch_stops_before_ble_recovery_and_vendor_commands(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, _ = service_with_fake_runtime(tmp_path)
|
||||
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
||||
service._connection_mode = "bridge" # noqa: SLF001
|
||||
service._k1_ip = "192.168.68.50" # noqa: SLF001
|
||||
service._device_id = "known-k1" # noqa: SLF001
|
||||
service._device_session_id = "known-session" # noqa: SLF001
|
||||
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||
|
||||
async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]:
|
||||
raise AssertionError("host route mismatch must stop before BLE recovery")
|
||||
|
||||
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: False)
|
||||
monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "tunnel")
|
||||
monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read)
|
||||
|
||||
with pytest.raises(
|
||||
facade_module.ConnectionLeaseUnavailable,
|
||||
match="другой локальной сети",
|
||||
):
|
||||
service.open_application_control_session(
|
||||
OpenApplicationControlSessionRequest(
|
||||
operator_present=True,
|
||||
owner_controlled_device=True,
|
||||
lixelgo_closed=True,
|
||||
battery_storage_confirmed=True,
|
||||
expected_physical_state_confirmed=True,
|
||||
timezone_name="Europe/Moscow",
|
||||
)
|
||||
)
|
||||
|
||||
state = service.state()
|
||||
assert state["connection_verification"]["reason_code"] == (
|
||||
"connection_lease_host_route_mismatch"
|
||||
)
|
||||
assert state["connection_verification"]["endpoint_validation"] == "host-route"
|
||||
assert state["connection_verification"]["host_route_class"] == "tunnel"
|
||||
assert state["application_control_session"]["state"] == "idle"
|
||||
|
||||
|
||||
def test_prepare_creates_provisional_device_session_and_profiled_acquisition(
|
||||
@@ -384,18 +621,24 @@ def test_acquisition_is_unbounded_by_default_and_accepts_ten_hour_hint() -> None
|
||||
|
||||
|
||||
def test_connection_modes_require_their_exact_topology_attestation() -> None:
|
||||
assert ConnectRequest(
|
||||
assert (
|
||||
ConnectRequest(
|
||||
device_id="synthetic-device",
|
||||
connection_mode="quick-connect",
|
||||
compatibility_attestation=QUICK_CONNECT_ATTESTATION,
|
||||
).connection_mode == "quick-connect"
|
||||
assert ConnectRequest(
|
||||
).connection_mode
|
||||
== "quick-connect"
|
||||
)
|
||||
assert (
|
||||
ConnectRequest(
|
||||
device_id="synthetic-device",
|
||||
ssid="synthetic-network",
|
||||
password=SecretStr(PRIMARY_TEST_CREDENTIAL),
|
||||
connection_mode="direct-connect",
|
||||
compatibility_attestation=DIRECT_CONNECT_ATTESTATION,
|
||||
).connection_mode == "direct-connect"
|
||||
).connection_mode
|
||||
== "direct-connect"
|
||||
)
|
||||
with pytest.raises(ValidationError):
|
||||
ConnectRequest(
|
||||
device_id="synthetic-device",
|
||||
@@ -628,6 +871,91 @@ def test_plugin_commanded_acquisition_keeps_start_and_stop_as_explicit_actions(
|
||||
assert completed["acquisition"]["result"]["device_state"] == "ready"
|
||||
assert completed["last_operation"]["status"] == "succeeded"
|
||||
assert runtime.stop_calls == 1
|
||||
assert completed["live_perception_shadow"]["active"] is False
|
||||
|
||||
|
||||
def test_next_scan_retires_stale_terminal_live_perception_ingress_before_start(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, runtime = service_with_fake_runtime(tmp_path)
|
||||
service.live_perception_ingress.begin_session("stale-completed-session")
|
||||
prepared = service.prepare_acquisition(
|
||||
PrepareAcquisitionRequest(
|
||||
project_name=PROJECT_NAME,
|
||||
host="192.168.1.20",
|
||||
compatibility_attestation=ATTESTATION,
|
||||
)
|
||||
)
|
||||
|
||||
state = service.start_acquisition(
|
||||
StartAcquisitionRequest(acquisition_id=prepared["acquisition"]["acquisition_id"])
|
||||
)
|
||||
|
||||
assert runtime.start_calls
|
||||
assert state["acquisition"]["state"] == "starting"
|
||||
assert state["live_perception_shadow"]["active"] is True
|
||||
assert state["live_perception_shadow"]["session_id"] != "stale-completed-session"
|
||||
|
||||
|
||||
def test_confirmed_scanning_activates_and_records_right_camera(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
service, runtime = service_with_fake_runtime(tmp_path)
|
||||
control = FakeInteractiveControlSession()
|
||||
service._application_control_session = control # type: ignore[assignment] # noqa: SLF001
|
||||
service._k1_ip = "192.168.1.20" # noqa: SLF001
|
||||
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||
|
||||
prepared = service.prepare_acquisition(
|
||||
PrepareAcquisitionRequest(
|
||||
project_name="TEST001",
|
||||
host="192.168.1.20",
|
||||
compatibility_attestation=ATTESTATION,
|
||||
)
|
||||
)
|
||||
acquisition_id = prepared["acquisition"]["acquisition_id"]
|
||||
service.start_acquisition(
|
||||
StartAcquisitionRequest(
|
||||
acquisition_id=acquisition_id,
|
||||
physical_acceptance=PHYSICAL_ACCEPTANCE,
|
||||
)
|
||||
)
|
||||
out_dir = service._acquisition_out_dir # noqa: SLF001
|
||||
assert out_dir is not None
|
||||
out_dir.mkdir(parents=True)
|
||||
events: list[tuple[str, object]] = []
|
||||
camera_state: dict[str, object] = {
|
||||
"phase": "idle",
|
||||
"active_source_id": None,
|
||||
"recording": {"active": False},
|
||||
}
|
||||
|
||||
def select_camera(source_id: str, target: str) -> dict[str, object]:
|
||||
events.append(("select", (source_id, target)))
|
||||
camera_state["phase"] = "selected"
|
||||
camera_state["active_source_id"] = source_id
|
||||
return dict(camera_state)
|
||||
|
||||
def start_recording(session_dir: Path) -> dict[str, object]:
|
||||
events.append(("record", session_dir))
|
||||
camera_state["recording"] = {"active": True}
|
||||
camera_state["phase"] = "connecting"
|
||||
return dict(camera_state)
|
||||
|
||||
monkeypatch.setattr(service.camera_preview, "snapshot", lambda: dict(camera_state))
|
||||
monkeypatch.setattr(service.camera_preview, "select", select_camera)
|
||||
monkeypatch.setattr(service.camera_preview, "start_recording", start_recording)
|
||||
|
||||
service._activate_default_acquisition_camera() # noqa: SLF001
|
||||
|
||||
assert events == [
|
||||
("select", ("sensor.camera.right", "192.168.1.20")),
|
||||
("record", out_dir),
|
||||
]
|
||||
assert camera_state["active_source_id"] == "sensor.camera.right"
|
||||
assert camera_state["recording"] == {"active": True}
|
||||
assert runtime.source_mode == "live"
|
||||
|
||||
|
||||
def test_device_standby_retires_sources_after_terminal_local_stop_failure(
|
||||
@@ -1611,6 +1939,9 @@ def test_runtime_error_terminalizes_pending_graceful_stop(tmp_path: Path) -> Non
|
||||
assert failed["acquisition"]["state"] == "failed"
|
||||
assert stop_operation["status"] == "failed"
|
||||
assert stop_operation["error"]["side_effect_status"] == "unknown"
|
||||
assert failed["source_mode"] == "idle"
|
||||
assert failed["phase"] != "error"
|
||||
assert runtime.stop_calls == 1
|
||||
|
||||
|
||||
def test_receiver_completion_terminalizes_unconfirmed_graceful_stop(
|
||||
@@ -1970,6 +2301,11 @@ def test_network_provisioning_is_single_flight_and_secret_is_unwrapped_only_at_b
|
||||
}
|
||||
|
||||
monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision)
|
||||
monkeypatch.setattr(
|
||||
facade_module,
|
||||
"_host_route_class",
|
||||
lambda _target: "direct-or-routed",
|
||||
)
|
||||
first = asyncio.create_task(
|
||||
service.connect(
|
||||
ConnectRequest(
|
||||
@@ -2005,6 +2341,117 @@ def test_network_provisioning_is_single_flight_and_secret_is_unwrapped_only_at_b
|
||||
assert {item["status"] for item in provision_operations} == {"succeeded", "failed"}
|
||||
|
||||
|
||||
def test_bridge_network_change_retires_terminal_acquisition_and_receiver_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, runtime = service_with_fake_runtime(tmp_path)
|
||||
service._devices = [{"device_id": "k1-a", "name": "XGR-TEST-A"}] # noqa: SLF001
|
||||
prepared = service.prepare_acquisition(
|
||||
PrepareAcquisitionRequest(
|
||||
project_name=PROJECT_NAME,
|
||||
host="192.168.1.20",
|
||||
compatibility_attestation=ATTESTATION,
|
||||
)
|
||||
)
|
||||
service.abort_acquisition(
|
||||
AbortAcquisitionRequest(
|
||||
acquisition_id=prepared["acquisition"]["acquisition_id"],
|
||||
)
|
||||
)
|
||||
runtime.phase = "error"
|
||||
runtime.source_mode = "live"
|
||||
stop_calls_before_connect = runtime.stop_calls
|
||||
|
||||
async def fake_provision(*_: object, **__: object) -> dict[str, Any]:
|
||||
return {
|
||||
"started_at_utc": "2026-07-28T18:11:37Z",
|
||||
"completed_at_utc": "2026-07-28T18:11:44Z",
|
||||
"profile_id": "xgrids-k1-fw3-wifi-v1",
|
||||
"outcome": "lan_address_observed",
|
||||
"observations": [{"status": {"ipv4": "192.168.1.20"}}],
|
||||
}
|
||||
|
||||
monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision)
|
||||
monkeypatch.setattr(
|
||||
facade_module,
|
||||
"_host_route_class",
|
||||
lambda _target: "direct-or-routed",
|
||||
)
|
||||
connected = asyncio.run(
|
||||
service.connect(
|
||||
ConnectRequest(
|
||||
device_id="k1-a",
|
||||
ssid="lab-network",
|
||||
password=SecretStr(PRIMARY_TEST_CREDENTIAL),
|
||||
connection_mode="bridge",
|
||||
compatibility_attestation=ATTESTATION,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert connected["connection_mode"] == "bridge"
|
||||
assert connected["k1_ip"] == "192.168.1.20"
|
||||
assert connected["acquisition"] is None
|
||||
assert connected["source_mode"] == "idle"
|
||||
assert connected["phase"] == "connected"
|
||||
assert runtime.stop_calls == stop_calls_before_connect + 1
|
||||
|
||||
|
||||
def test_bridge_provisioning_reports_host_route_mismatch_without_hiding_device_success(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, _ = service_with_fake_runtime(tmp_path)
|
||||
service._devices = [{"device_id": "k1-a", "name": "XGR-TEST-A"}] # noqa: SLF001
|
||||
|
||||
async def fake_provision(*_: object, **__: object) -> dict[str, Any]:
|
||||
return {
|
||||
"started_at_utc": "2026-07-28T19:43:03Z",
|
||||
"completed_at_utc": "2026-07-28T19:43:16Z",
|
||||
"profile_id": "xgrids-k1-fw3-wifi-v1",
|
||||
"outcome": "lan_address_observed",
|
||||
"observations": [{"status": {"ipv4": "192.168.68.50"}}],
|
||||
}
|
||||
|
||||
monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision)
|
||||
monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "tunnel")
|
||||
|
||||
connected = asyncio.run(
|
||||
service.connect(
|
||||
ConnectRequest(
|
||||
device_id="k1-a",
|
||||
ssid="lab-router",
|
||||
password=SecretStr(PRIMARY_TEST_CREDENTIAL),
|
||||
connection_mode="bridge",
|
||||
compatibility_attestation=ATTESTATION,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert connected["k1_ip"] == "192.168.68.50"
|
||||
assert connected["phase"] == "device_selected"
|
||||
assert "компьютер подключён к другой сети" in connected["message"]
|
||||
assert connected["device_session"]["connectivity"] == "offline"
|
||||
assert connected["connection_verification"] == {
|
||||
"status": "host-route-mismatch",
|
||||
"lease_state": "disconnected",
|
||||
"lease_generation": 1,
|
||||
"endpoint_validation": "host-route",
|
||||
"network_reachability": "unreachable",
|
||||
"reason_code": "connection_lease_host_route_mismatch",
|
||||
"host_route_class": "tunnel",
|
||||
"write_performed": True,
|
||||
"observed_at": connected["connection_verification"]["observed_at"],
|
||||
}
|
||||
operation = next(
|
||||
item for item in connected["operations"] if item["action"] == "network.provision"
|
||||
)
|
||||
assert operation["status"] == "succeeded"
|
||||
assert operation["result"]["host_route_ready"] is False
|
||||
assert operation["result"]["host_route_class"] == "tunnel"
|
||||
|
||||
|
||||
def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
@@ -2028,9 +2475,7 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_activation_session(
|
||||
device_id: str, **_: object
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
async def fake_activation_session(device_id: str, **_: object) -> AsyncIterator[dict[str, Any]]:
|
||||
nonlocal ble_session_open
|
||||
activation_calls.append(device_id)
|
||||
ble_session_open = True
|
||||
@@ -2092,9 +2537,7 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
||||
assert not ble_session_open
|
||||
assert len(association_calls) == 1
|
||||
assert association_calls[0][0].name == "associate_wifi.swift"
|
||||
assert association_calls[0][1] == facade_module.quick_connect_host_profile_id(
|
||||
"XGR-TEST-A"
|
||||
)
|
||||
assert association_calls[0][1] == facade_module.quick_connect_host_profile_id("XGR-TEST-A")
|
||||
assert association_calls[0][2] == "XGR-TEST-A"
|
||||
assert state["connection_mode"] == "quick-connect"
|
||||
assert state["k1_ip"] == "192.168.56.1"
|
||||
@@ -2102,9 +2545,7 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
||||
quick_sessions = sorted(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
||||
assert len(quick_sessions) == 1
|
||||
assert not (quick_sessions[0] / "provisioning.sensitive.json").exists()
|
||||
redacted_manifest = (quick_sessions[0] / "manifest.redacted.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
redacted_manifest = (quick_sessions[0] / "manifest.redacted.json").read_text(encoding="utf-8")
|
||||
assert PRIMARY_TEST_CREDENTIAL not in redacted_manifest
|
||||
assert "host_wifi_profile_id" in redacted_manifest
|
||||
assert '"host_wifi_profile_ready_before_device_write": true' in redacted_manifest
|
||||
@@ -2112,9 +2553,12 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
||||
assert "credential_provider_id" in redacted_manifest
|
||||
assert "device_ap_activation_profile_id" in redacted_manifest
|
||||
assert (quick_sessions[0] / "ap-activation.redacted.json").exists()
|
||||
assert service._camera_target_for_session( # noqa: SLF001
|
||||
assert (
|
||||
service._camera_target_for_session( # noqa: SLF001
|
||||
state["device_session"]["device_session_id"]
|
||||
) == "192.168.56.1"
|
||||
)
|
||||
== "192.168.56.1"
|
||||
)
|
||||
|
||||
prepared = service.prepare_acquisition(
|
||||
PrepareAcquisitionRequest(
|
||||
@@ -2158,6 +2602,11 @@ def test_direct_connect_reuses_the_reviewed_ble_provisioning_frame(
|
||||
forbidden_host_association,
|
||||
)
|
||||
monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False)
|
||||
monkeypatch.setattr(
|
||||
facade_module,
|
||||
"_host_route_class",
|
||||
lambda _target: "direct-or-routed",
|
||||
)
|
||||
|
||||
state = asyncio.run(
|
||||
service.connect(
|
||||
@@ -2171,14 +2620,10 @@ def test_direct_connect_reuses_the_reviewed_ble_provisioning_frame(
|
||||
)
|
||||
)
|
||||
|
||||
assert provisioning_calls == [
|
||||
("k1-a", "controller-hotspot", PRIMARY_TEST_CREDENTIAL)
|
||||
]
|
||||
assert provisioning_calls == [("k1-a", "controller-hotspot", PRIMARY_TEST_CREDENTIAL)]
|
||||
assert state["connection_mode"] == "direct-connect"
|
||||
assert state["k1_ip"] == "172.20.10.2"
|
||||
assert state["compatibility"]["attestation"]["topology"] == (
|
||||
"controller-hotspot"
|
||||
)
|
||||
assert state["compatibility"]["attestation"]["topology"] == ("controller-hotspot")
|
||||
|
||||
|
||||
def test_quick_connect_missing_credential_provider_stops_before_ap_write(
|
||||
@@ -2232,9 +2677,7 @@ def test_quick_connect_missing_credential_provider_stops_before_ap_write(
|
||||
assert state["k1_ip"] == "192.168.1.20"
|
||||
assert state["connection_mode"] == "bridge"
|
||||
assert not list(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
||||
operation = next(
|
||||
item for item in state["operations"] if item["action"] == "network.provision"
|
||||
)
|
||||
operation = next(item for item in state["operations"] if item["action"] == "network.provision")
|
||||
assert operation["status"] == "failed"
|
||||
assert operation["error"]["side_effect_status"] == "none"
|
||||
assert operation["error"]["safe_to_retry"] is True
|
||||
@@ -2259,9 +2702,7 @@ def test_quick_connect_does_not_start_host_wifi_without_ap_ready_flag(
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def not_ready_session(
|
||||
*_: object, **__: object
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
async def not_ready_session(*_: object, **__: object) -> AsyncIterator[dict[str, Any]]:
|
||||
yield {
|
||||
"profile_id": "xgrids-k1-fw3-quick-connect-ap-v1",
|
||||
"started_at_utc": "2026-07-19T15:00:00Z",
|
||||
@@ -2325,9 +2766,7 @@ def test_failed_connection_change_revokes_the_previous_route(
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_activation_session(
|
||||
*_: object, **__: object
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
async def fake_activation_session(*_: object, **__: object) -> AsyncIterator[dict[str, Any]]:
|
||||
yield {
|
||||
"profile_id": "xgrids-k1-fw3-quick-connect-ap-v1",
|
||||
"started_at_utc": "2026-07-19T15:00:00Z",
|
||||
@@ -2370,9 +2809,7 @@ def test_failed_connection_change_revokes_the_previous_route(
|
||||
assert state["compatibility"]["attestation"] is None
|
||||
quick_sessions = sorted(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
||||
failure_evidence = json.loads(
|
||||
(quick_sessions[0] / "host-wifi-association.redacted.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
(quick_sessions[0] / "host-wifi-association.redacted.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert failure_evidence["reason_code"] == "network-not-found"
|
||||
assert failure_evidence["scan_attempt_count"] == 4
|
||||
@@ -2413,9 +2850,7 @@ def test_network_provisioning_rejects_an_ipv4_owned_by_the_local_host(
|
||||
state = service.state()
|
||||
assert state["selected_device_id"] is None
|
||||
assert state["k1_ip"] is None
|
||||
operation = next(
|
||||
item for item in state["operations"] if item["action"] == "network.provision"
|
||||
)
|
||||
operation = next(item for item in state["operations"] if item["action"] == "network.provision")
|
||||
assert operation["status"] == "failed"
|
||||
assert operation["error"]["safe_to_retry"] is False
|
||||
assert operation["error"]["side_effect_status"] == "unknown"
|
||||
|
||||
@@ -778,3 +778,36 @@ def test_post_publish_timeout_poisoned_transport_never_retries() -> None:
|
||||
required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"},
|
||||
)
|
||||
assert len(fake.publish_calls) == 1
|
||||
|
||||
|
||||
def test_network_loop_failure_keeps_exact_paho_result_and_phase() -> None:
|
||||
class LoopFailureClient(FakeControlClient):
|
||||
def loop(self, timeout: float) -> mqtt.MQTTErrorCode:
|
||||
if self.publish_calls and not self.events:
|
||||
return mqtt.MQTT_ERR_CONN_LOST
|
||||
return super().loop(timeout)
|
||||
|
||||
fake = LoopFailureClient()
|
||||
transport = ReviewedApplicationMqttTransport(
|
||||
"192.168.1.20",
|
||||
client_factory=lambda: cast(mqtt.Client, fake),
|
||||
)
|
||||
transport.open()
|
||||
|
||||
with pytest.raises(
|
||||
ApplicationCommandOutcomeUnknown,
|
||||
match=r"phase=post-publish-drain, result=7: The connection was lost",
|
||||
):
|
||||
transport.exchange_batch_once(
|
||||
[_envelope()],
|
||||
required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"},
|
||||
)
|
||||
|
||||
snapshot = transport.snapshot().as_dict()
|
||||
assert snapshot["state"] == "poisoned"
|
||||
assert snapshot["last_loop_result_code"] == int(mqtt.MQTT_ERR_CONN_LOST)
|
||||
assert snapshot["last_loop_result_name"] == mqtt.error_string(
|
||||
int(mqtt.MQTT_ERR_CONN_LOST)
|
||||
)
|
||||
assert snapshot["last_loop_phase"] == "post-publish-drain"
|
||||
assert len(fake.publish_calls) == 1
|
||||
|
||||
@@ -170,6 +170,7 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
FakeExecutor.records = []
|
||||
scanning_observed = threading.Event()
|
||||
monkeypatch.setattr(
|
||||
session_module,
|
||||
"PhysicalAcceptanceDialogueExecutor",
|
||||
@@ -181,6 +182,7 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not(
|
||||
loader,
|
||||
transport_factory=lambda _host: transport, # type: ignore[arg-type]
|
||||
epoch_seconds=lambda: 1_752_680_000,
|
||||
scanning_observer=scanning_observed.set,
|
||||
)
|
||||
|
||||
session.open(
|
||||
@@ -202,6 +204,7 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not(
|
||||
|
||||
session.request_start(project_name="TEST001", confirmation=_confirmation())
|
||||
_wait_phase(session, "scanning")
|
||||
assert scanning_observed.wait(timeout=1.0)
|
||||
assert FakeExecutor.records[-1] == "wait:stop"
|
||||
|
||||
session.request_stop(confirmation=_confirmation())
|
||||
@@ -210,6 +213,7 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not(
|
||||
assert completed["pending_operator_action"] is None
|
||||
assert loader.calls == 1
|
||||
assert completed["automatic_retry"] is False
|
||||
assert completed["scanning_observer_errors"] == 0
|
||||
assert completed["scripted_transitions"] is False
|
||||
assert FakeExecutor.records == [
|
||||
"connection:1-6",
|
||||
@@ -562,6 +566,210 @@ def test_start_outcome_unknown_blocks_reopen_even_when_transport_is_closed(
|
||||
assert failed["can_open"] is False
|
||||
assert failed["failure"]["modeling_command_attempted"] is True # type: ignore[index]
|
||||
assert failed["failure"]["safe_to_retry"] is False # type: ignore[index]
|
||||
with pytest.raises(
|
||||
session_module.ApplicationAcceptanceError,
|
||||
match="cannot be retired",
|
||||
):
|
||||
session.retire_for_network_change()
|
||||
|
||||
|
||||
def test_acknowledged_stop_disconnect_allows_only_explicit_network_change(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
@dataclass
|
||||
class StopDisconnectTransportSnapshot:
|
||||
state: str
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"state": self.state,
|
||||
"publish_attempts": 15,
|
||||
"qos2_completions": 15,
|
||||
"correlated_responses": 13,
|
||||
"ignored_known_responses": 304,
|
||||
"late_known_responses": 0,
|
||||
"device_status_reports": 52,
|
||||
"latest_device_session_state": "scan_stopping",
|
||||
"latest_device_project_bound": True,
|
||||
"latest_system_error_code": None,
|
||||
"last_loop_result_code": 7,
|
||||
"last_loop_result_name": "The connection was lost.",
|
||||
"last_loop_phase": "maintain-open",
|
||||
"automatic_retry": False,
|
||||
"automatic_reconnect": False,
|
||||
}
|
||||
|
||||
class StopDisconnectTransport(FakeTransport):
|
||||
def snapshot(self) -> StopDisconnectTransportSnapshot:
|
||||
return StopDisconnectTransportSnapshot(self.state)
|
||||
|
||||
class StopDisconnectExecutor(FakeExecutor):
|
||||
def __init__(self, transport: StopDisconnectTransport) -> None:
|
||||
super().__init__(transport)
|
||||
self.stop_attempted = False
|
||||
self.stop_complete = False
|
||||
|
||||
def execute_canonical_start(self, *_args: object, **_kwargs: object) -> object:
|
||||
self.records.append("start:11-14")
|
||||
return object()
|
||||
|
||||
def execute_canonical_stop(self, *_args: object, **_kwargs: object) -> object:
|
||||
self.records.append("stop")
|
||||
self.stop_attempted = True
|
||||
self.stop_complete = True
|
||||
return object()
|
||||
|
||||
def maintain_post_stop_until_standby(self) -> None:
|
||||
self.records.append("wait:device-standby")
|
||||
self.transport.state = "poisoned"
|
||||
raise session_module.ApplicationCommandOutcomeUnknown(
|
||||
"control MQTT network loop returned an error",
|
||||
reason_code="mqtt_network_loop_failed",
|
||||
)
|
||||
|
||||
def snapshot(self) -> dict[str, object]:
|
||||
return {
|
||||
"dialogue_stage": (
|
||||
"stop-acknowledged" if self.stop_complete else "test-stage"
|
||||
),
|
||||
"start_attempted": True,
|
||||
"start_complete": True,
|
||||
"stop_attempted": self.stop_attempted,
|
||||
"stop_complete": self.stop_complete,
|
||||
"response_evidence": [],
|
||||
"automatic_retry": False,
|
||||
}
|
||||
|
||||
FakeExecutor.records = []
|
||||
monkeypatch.setattr(
|
||||
session_module,
|
||||
"PhysicalAcceptanceDialogueExecutor",
|
||||
StopDisconnectExecutor,
|
||||
)
|
||||
session = InteractiveApplicationControlSession(
|
||||
FakeAuthorityLoader(),
|
||||
transport_factory=lambda host: StopDisconnectTransport(host), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
session.open(
|
||||
host="192.168.1.20",
|
||||
timezone_name="Europe/Moscow",
|
||||
confirmation=_confirmation(),
|
||||
)
|
||||
_wait_phase(session, "connection-ready")
|
||||
session.enter_workspace()
|
||||
_wait_phase(session, "workspace-ready")
|
||||
session.open_project_prompt()
|
||||
_wait_phase(session, "project-ready")
|
||||
session.request_start(project_name="TEST001", confirmation=_confirmation())
|
||||
_wait_phase(session, "scanning")
|
||||
session.request_stop(confirmation=_confirmation())
|
||||
failed = _wait_phase(session, "failed")
|
||||
failed_thread = session._thread # noqa: SLF001
|
||||
assert failed_thread is not None
|
||||
failed_thread.join(timeout=2.0)
|
||||
assert not failed_thread.is_alive()
|
||||
failed = session.snapshot()
|
||||
|
||||
assert failed["can_open"] is False
|
||||
assert failed["outcome_unknown"] is True
|
||||
failure = failed["failure"]
|
||||
assert isinstance(failure, dict)
|
||||
assert failure["safe_to_retry"] is False
|
||||
assert failure["network_change_admissible"] is True
|
||||
assert failure["network_change_reconciliation"] == {
|
||||
"device_session_state": "scan_stopping",
|
||||
"device_project_bound": True,
|
||||
"system_error_code": None,
|
||||
"stop_complete": True,
|
||||
"standby_confirmed": False,
|
||||
"decision": "explicit-network-change-only-after-acknowledged-stop",
|
||||
"automatic_retry": False,
|
||||
}
|
||||
|
||||
retired = session.retire_for_network_change()
|
||||
assert retired["state"] == "idle"
|
||||
assert retired["failure"] is None
|
||||
assert retired["can_open"] is True
|
||||
|
||||
|
||||
def test_prestart_loop_failure_allows_only_fresh_explicit_retry_after_ready_status(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
@dataclass
|
||||
class ReconciledTransportSnapshot:
|
||||
state: str
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"state": self.state,
|
||||
"publish_attempts": 4,
|
||||
"qos2_completions": 4,
|
||||
"correlated_responses": 3,
|
||||
"ignored_known_responses": 0,
|
||||
"late_known_responses": 0,
|
||||
"device_status_reports": 1,
|
||||
"latest_device_session_state": "ready",
|
||||
"latest_device_project_bound": False,
|
||||
"latest_system_error_code": None,
|
||||
"last_loop_result_code": 7,
|
||||
"last_loop_result_name": "The connection was lost.",
|
||||
"last_loop_phase": "post-publish-drain",
|
||||
"automatic_retry": False,
|
||||
"automatic_reconnect": False,
|
||||
}
|
||||
|
||||
class ReconciledTransport(FakeTransport):
|
||||
def snapshot(self) -> ReconciledTransportSnapshot:
|
||||
return ReconciledTransportSnapshot(self.state)
|
||||
|
||||
class PrestartLoopFailureExecutor(FakeExecutor):
|
||||
def run_connection_stage(
|
||||
self,
|
||||
_orchestrator: object,
|
||||
) -> LiveDeviceControlBinding:
|
||||
raise session_module.ApplicationCommandOutcomeUnknown(
|
||||
"control MQTT network loop returned an error",
|
||||
reason_code="mqtt_network_loop_failed",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
session_module,
|
||||
"PhysicalAcceptanceDialogueExecutor",
|
||||
PrestartLoopFailureExecutor,
|
||||
)
|
||||
session = InteractiveApplicationControlSession(
|
||||
FakeAuthorityLoader(),
|
||||
transport_factory=lambda host: ReconciledTransport(host), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
session.open(
|
||||
host="192.168.1.20",
|
||||
timezone_name="Europe/Moscow",
|
||||
confirmation=_confirmation(),
|
||||
)
|
||||
failed_thread = session._thread # noqa: SLF001
|
||||
assert failed_thread is not None
|
||||
failed_thread.join(timeout=2.0)
|
||||
assert not failed_thread.is_alive()
|
||||
failed = session.snapshot()
|
||||
|
||||
assert failed["state"] == "failed"
|
||||
assert failed["outcome_unknown"] is True
|
||||
assert failed["automatic_retry"] is False
|
||||
assert failed["can_open"] is True
|
||||
failure = failed["failure"]
|
||||
assert isinstance(failure, dict)
|
||||
assert failure["reason_code"] == "mqtt_network_loop_failed"
|
||||
assert failure["modeling_command_attempted"] is False
|
||||
assert failure["safe_to_retry"] is True
|
||||
assert failure["status_reconciliation"] == {
|
||||
"device_session_state": "ready",
|
||||
"device_project_bound": False,
|
||||
"system_error_code": None,
|
||||
"decision": "safe-explicit-prestart-retry",
|
||||
"automatic_retry": False,
|
||||
}
|
||||
|
||||
|
||||
def test_unavailable_transport_snapshot_after_publish_blocks_reopen(
|
||||
|
||||
Reference in New Issue
Block a user