fix(k1): stabilize repeated acquisition and live viewer recovery
This commit is contained in:
@@ -1,6 +1,15 @@
|
|||||||
import { useEffect, useRef, useState } from "react";
|
import { useEffect, useRef, useState } from "react";
|
||||||
|
|
||||||
import type { SceneSettings } from "../sceneSettings";
|
import type { SceneSettings } from "../sceneSettings";
|
||||||
|
import {
|
||||||
|
advanceLiveReceiverWatchdog,
|
||||||
|
initialLiveReceiverRecoveryState,
|
||||||
|
initialLiveReceiverWatchdogState,
|
||||||
|
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
||||||
|
requestLiveReceiverRecovery,
|
||||||
|
} from "../core/observation/liveReceiverWatchdog";
|
||||||
|
import { postLiveViewerDiagnostic } from "../core/observation/liveViewerDiagnostics";
|
||||||
|
import type { LiveViewerFailureStage } from "../core/observation/liveViewerDiagnostics";
|
||||||
import type { RecordedAdmissionPhase } from "../core/observation/recordedSessionAdmission";
|
import type { RecordedAdmissionPhase } from "../core/observation/recordedSessionAdmission";
|
||||||
|
|
||||||
export type RerunViewportStatus = "idle" | "loading" | "ready" | "error";
|
export type RerunViewportStatus = "idle" | "loading" | "ready" | "error";
|
||||||
@@ -66,6 +75,8 @@ export interface RerunViewportProps {
|
|||||||
sourceUrl: string;
|
sourceUrl: string;
|
||||||
recordedArtifact?: RecordedRrdArtifactDescriptor | null;
|
recordedArtifact?: RecordedRrdArtifactDescriptor | null;
|
||||||
followLive?: boolean;
|
followLive?: boolean;
|
||||||
|
liveActivitySequence?: number | null;
|
||||||
|
liveStreamId?: string | null;
|
||||||
autoplayWhenReady?: boolean;
|
autoplayWhenReady?: boolean;
|
||||||
presentationGate?: RecordedAdmissionPhase;
|
presentationGate?: RecordedAdmissionPhase;
|
||||||
expectedTimelineStartSeconds?: number;
|
expectedTimelineStartSeconds?: number;
|
||||||
@@ -796,6 +807,8 @@ export function RerunViewport({
|
|||||||
sourceUrl,
|
sourceUrl,
|
||||||
recordedArtifact = null,
|
recordedArtifact = null,
|
||||||
followLive = false,
|
followLive = false,
|
||||||
|
liveActivitySequence = null,
|
||||||
|
liveStreamId = null,
|
||||||
autoplayWhenReady = false,
|
autoplayWhenReady = false,
|
||||||
presentationGate = "ready",
|
presentationGate = "ready",
|
||||||
expectedTimelineStartSeconds,
|
expectedTimelineStartSeconds,
|
||||||
@@ -824,6 +837,11 @@ export function RerunViewport({
|
|||||||
const [status, setStatus] = useState<RerunViewportStatus>(sourceUrl ? "loading" : "idle");
|
const [status, setStatus] = useState<RerunViewportStatus>(sourceUrl ? "loading" : "idle");
|
||||||
const [recordingBufferProgress, setRecordingBufferProgress] = useState<number | null>(null);
|
const [recordingBufferProgress, setRecordingBufferProgress] = useState<number | null>(null);
|
||||||
const [retryNonce, setRetryNonce] = useState(0);
|
const [retryNonce, setRetryNonce] = useState(0);
|
||||||
|
const liveActivitySequenceRef = useRef<number | null>(liveActivitySequence);
|
||||||
|
liveActivitySequenceRef.current = liveActivitySequence;
|
||||||
|
const liveStreamIdRef = useRef<string | null>(liveStreamId);
|
||||||
|
liveStreamIdRef.current = liveStreamId;
|
||||||
|
const liveRecoveryRef = useRef(initialLiveReceiverRecoveryState());
|
||||||
const blueprintChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
const blueprintChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
||||||
const perceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
const perceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
||||||
const loadedPerceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
const loadedPerceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
||||||
@@ -849,6 +867,10 @@ export function RerunViewport({
|
|||||||
recordedArtifact !== null,
|
recordedArtifact !== null,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
liveRecoveryRef.current = initialLiveReceiverRecoveryState();
|
||||||
|
}, [followLive, liveStreamId, sourceUrl]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const normalizedSource = sourceUrl.trim();
|
const normalizedSource = sourceUrl.trim();
|
||||||
const host = hostRef.current;
|
const host = hostRef.current;
|
||||||
@@ -889,6 +911,7 @@ export function RerunViewport({
|
|||||||
let disposed = false;
|
let disposed = false;
|
||||||
let disposeViewer: (() => void) | undefined;
|
let disposeViewer: (() => void) | undefined;
|
||||||
let recordingOpenTimer: number | undefined;
|
let recordingOpenTimer: number | undefined;
|
||||||
|
let liveRecordingDiscoveryTimer: number | undefined;
|
||||||
let recordedOpenWatchdog: {
|
let recordedOpenWatchdog: {
|
||||||
arm: () => void;
|
arm: () => void;
|
||||||
clear: () => void;
|
clear: () => void;
|
||||||
@@ -898,6 +921,8 @@ export function RerunViewport({
|
|||||||
let recordingOpened = false;
|
let recordingOpened = false;
|
||||||
let recordingOpenTimedOut = false;
|
let recordingOpenTimedOut = false;
|
||||||
let viewerStartResolved = false;
|
let viewerStartResolved = false;
|
||||||
|
let latestLiveRangeMaxNs: number | null = null;
|
||||||
|
let liveWatchdog = initialLiveReceiverWatchdogState();
|
||||||
// The backend serves this exact digest-bound URL or rejects it with 412.
|
// The backend serves this exact digest-bound URL or rejects it with 412.
|
||||||
// Presentation still waits for Rerun to decode the complete declared range.
|
// Presentation still waits for Rerun to decode the complete declared range.
|
||||||
const artifactVerified = true;
|
const artifactVerified = true;
|
||||||
@@ -925,9 +950,16 @@ export function RerunViewport({
|
|||||||
recordingOpenTimer = undefined;
|
recordingOpenTimer = undefined;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
const clearLiveRecordingDiscoveryTimer = () => {
|
||||||
|
if (liveRecordingDiscoveryTimer !== undefined) {
|
||||||
|
window.clearInterval(liveRecordingDiscoveryTimer);
|
||||||
|
liveRecordingDiscoveryTimer = undefined;
|
||||||
|
}
|
||||||
|
};
|
||||||
const clearRecordingTimers = () => {
|
const clearRecordingTimers = () => {
|
||||||
clearRecordedAdmissionWatchdog();
|
clearRecordedAdmissionWatchdog();
|
||||||
clearLiveRecordingOpenTimer();
|
clearLiveRecordingOpenTimer();
|
||||||
|
clearLiveRecordingDiscoveryTimer();
|
||||||
};
|
};
|
||||||
const clearPlaybackRangeTimer = () => {
|
const clearPlaybackRangeTimer = () => {
|
||||||
if (playbackRangeTimer === undefined) return;
|
if (playbackRangeTimer === undefined) return;
|
||||||
@@ -950,11 +982,118 @@ export function RerunViewport({
|
|||||||
requestFrame: (callback) => window.requestAnimationFrame(callback),
|
requestFrame: (callback) => window.requestAnimationFrame(callback),
|
||||||
cancelFrame: (handle) => window.cancelAnimationFrame(handle),
|
cancelFrame: (handle) => window.cancelAnimationFrame(handle),
|
||||||
});
|
});
|
||||||
const reportError = (message: string) => {
|
const reportError = (message: string, failureStage?: LiveViewerFailureStage) => {
|
||||||
if (disposed) return;
|
if (disposed) return;
|
||||||
|
if (followLive) {
|
||||||
|
postLiveViewerDiagnostic({
|
||||||
|
eventCode: "live_receiver_error",
|
||||||
|
failureStage,
|
||||||
|
streamId: liveStreamIdRef.current,
|
||||||
|
backendActivitySequence: liveActivitySequenceRef.current,
|
||||||
|
viewerRangeMaxNs: latestLiveRangeMaxNs,
|
||||||
|
recoveryAttempt: liveRecoveryRef.current.attempts || null,
|
||||||
|
});
|
||||||
|
}
|
||||||
setStatus("error");
|
setStatus("error");
|
||||||
onStatusChange?.("error", message);
|
onStatusChange?.("error", message);
|
||||||
};
|
};
|
||||||
|
const requestLiveRecovery = (
|
||||||
|
failureStage: LiveViewerFailureStage,
|
||||||
|
stalledForMs = 0,
|
||||||
|
emitErrorEvent = true,
|
||||||
|
) => {
|
||||||
|
if (!followLive || disposed) return false;
|
||||||
|
if (emitErrorEvent) {
|
||||||
|
postLiveViewerDiagnostic({
|
||||||
|
eventCode: "live_receiver_error",
|
||||||
|
failureStage,
|
||||||
|
streamId: liveStreamIdRef.current,
|
||||||
|
backendActivitySequence: liveActivitySequenceRef.current,
|
||||||
|
viewerRangeMaxNs: latestLiveRangeMaxNs,
|
||||||
|
stalledForMs: Math.round(stalledForMs) || null,
|
||||||
|
recoveryAttempt: liveRecoveryRef.current.attempts || null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const recovery = requestLiveReceiverRecovery(liveRecoveryRef.current);
|
||||||
|
liveRecoveryRef.current = recovery.state;
|
||||||
|
if (recovery.signal === "exhausted") {
|
||||||
|
postLiveViewerDiagnostic({
|
||||||
|
eventCode: "live_receiver_recovery_exhausted",
|
||||||
|
failureStage,
|
||||||
|
streamId: liveStreamIdRef.current,
|
||||||
|
backendActivitySequence: liveActivitySequenceRef.current,
|
||||||
|
viewerRangeMaxNs: latestLiveRangeMaxNs,
|
||||||
|
stalledForMs: Math.round(stalledForMs) || null,
|
||||||
|
recoveryAttempt: LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
||||||
|
});
|
||||||
|
setStatus("error");
|
||||||
|
onStatusChange?.(
|
||||||
|
"error",
|
||||||
|
"Живой поток продолжает поступать, но визуализатор не восстановился после трёх переподключений.",
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
postLiveViewerDiagnostic({
|
||||||
|
eventCode: "live_receiver_restart_requested",
|
||||||
|
failureStage,
|
||||||
|
streamId: liveStreamIdRef.current,
|
||||||
|
backendActivitySequence: liveActivitySequenceRef.current,
|
||||||
|
viewerRangeMaxNs: latestLiveRangeMaxNs,
|
||||||
|
stalledForMs: Math.round(stalledForMs) || null,
|
||||||
|
recoveryAttempt: recovery.attempt,
|
||||||
|
});
|
||||||
|
setStatus("loading");
|
||||||
|
onStatusChange?.(
|
||||||
|
"loading",
|
||||||
|
"Живой визуализатор переподключается к продолжающемуся потоку.",
|
||||||
|
);
|
||||||
|
disposeViewer?.();
|
||||||
|
setRetryNonce((nonce) => nonce + 1);
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
const observeLiveReceiver = (viewerRangeMaxNs: number | null) => {
|
||||||
|
if (!followLive || disposed) return;
|
||||||
|
latestLiveRangeMaxNs = viewerRangeMaxNs;
|
||||||
|
const receiverOpenedAfterRecovery =
|
||||||
|
liveRecoveryRef.current.awaitingRecovery &&
|
||||||
|
liveWatchdog.lastViewerRangeMaxNs === null &&
|
||||||
|
viewerRangeMaxNs !== null;
|
||||||
|
const observed = advanceLiveReceiverWatchdog(liveWatchdog, {
|
||||||
|
nowMs: Date.now(),
|
||||||
|
backendActivitySequence: liveActivitySequenceRef.current,
|
||||||
|
viewerRangeMaxNs,
|
||||||
|
});
|
||||||
|
liveWatchdog = observed.state;
|
||||||
|
if (
|
||||||
|
(receiverOpenedAfterRecovery || observed.signal === "receiver-advanced") &&
|
||||||
|
liveRecoveryRef.current.awaitingRecovery
|
||||||
|
) {
|
||||||
|
postLiveViewerDiagnostic({
|
||||||
|
eventCode: "live_receiver_recovered",
|
||||||
|
streamId: liveStreamIdRef.current,
|
||||||
|
backendActivitySequence: liveActivitySequenceRef.current,
|
||||||
|
viewerRangeMaxNs,
|
||||||
|
recoveryAttempt: liveRecoveryRef.current.attempts,
|
||||||
|
});
|
||||||
|
liveRecoveryRef.current = initialLiveReceiverRecoveryState();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (observed.signal !== "stalled") return;
|
||||||
|
|
||||||
|
postLiveViewerDiagnostic({
|
||||||
|
eventCode: "live_receiver_stalled",
|
||||||
|
failureStage: "receiver-stalled",
|
||||||
|
streamId: liveStreamIdRef.current,
|
||||||
|
backendActivitySequence: liveActivitySequenceRef.current,
|
||||||
|
viewerRangeMaxNs,
|
||||||
|
stalledForMs: Math.round(observed.stalledForMs),
|
||||||
|
recoveryAttempt: Math.min(
|
||||||
|
liveRecoveryRef.current.attempts + 1,
|
||||||
|
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
||||||
|
),
|
||||||
|
});
|
||||||
|
requestLiveRecovery("receiver-stalled", observed.stalledForMs, false);
|
||||||
|
};
|
||||||
|
|
||||||
host.replaceChildren();
|
host.replaceChildren();
|
||||||
appliedPointColorKeyRef.current = null;
|
appliedPointColorKeyRef.current = null;
|
||||||
@@ -1016,7 +1155,7 @@ export function RerunViewport({
|
|||||||
viewer.stop();
|
viewer.stop();
|
||||||
} catch {
|
} catch {
|
||||||
// A partially initialized WASM handle can already be gone after a
|
// A partially initialized WASM handle can already be gone after a
|
||||||
// startup failure. The host still has to be cleared below.
|
// startup failure.
|
||||||
}
|
}
|
||||||
host.replaceChildren();
|
host.replaceChildren();
|
||||||
};
|
};
|
||||||
@@ -1034,14 +1173,20 @@ export function RerunViewport({
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
unsubscribers.push(
|
const admitRecording = (event: {
|
||||||
viewer.on("recording_open", (event) => {
|
application_id: string;
|
||||||
|
recording_id: string;
|
||||||
|
}) => {
|
||||||
if (
|
if (
|
||||||
disposed ||
|
disposed ||
|
||||||
recordingOpened ||
|
recordingOpened ||
|
||||||
(isRecordedSource && event.application_id !== "nodedc_mission_core_recorded")
|
(isRecordedSource && event.application_id !== "nodedc_mission_core_recorded")
|
||||||
) return;
|
) return;
|
||||||
recordingOpened = true;
|
recordingOpened = true;
|
||||||
|
if (!isRecordedSource) {
|
||||||
|
clearLiveRecordingOpenTimer();
|
||||||
|
clearLiveRecordingDiscoveryTimer();
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
recordedBlueprintUrl &&
|
recordedBlueprintUrl &&
|
||||||
event.application_id === "nodedc_mission_core_recorded" &&
|
event.application_id === "nodedc_mission_core_recorded" &&
|
||||||
@@ -1060,7 +1205,6 @@ export function RerunViewport({
|
|||||||
setPerceptionChannelRevision((revision) => revision + 1);
|
setPerceptionChannelRevision((revision) => revision + 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!isRecordedSource) clearLiveRecordingOpenTimer();
|
|
||||||
// Durable Mission Core recordings use a zero-based session clock.
|
// Durable Mission Core recordings use a zero-based session clock.
|
||||||
// Keeping wall-clock capture_time as a secondary RRD timeline is
|
// Keeping wall-clock capture_time as a secondary RRD timeline is
|
||||||
// useful for audit, but the operator scrubber must remain a small,
|
// useful for audit, but the operator scrubber must remain a small,
|
||||||
@@ -1144,6 +1288,7 @@ export function RerunViewport({
|
|||||||
} catch {
|
} catch {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
observeLiveReceiver(rangeNs?.max ?? null);
|
||||||
const recordedBuffer = recordedPlaybackBufferState(
|
const recordedBuffer = recordedPlaybackBufferState(
|
||||||
rangeNs,
|
rangeNs,
|
||||||
expectedTimelineEndSeconds,
|
expectedTimelineEndSeconds,
|
||||||
@@ -1225,8 +1370,31 @@ export function RerunViewport({
|
|||||||
clearPlaybackRangeTimer();
|
clearPlaybackRangeTimer();
|
||||||
publishBufferedState();
|
publishBufferedState();
|
||||||
playbackRangeTimer = window.setInterval(publishBufferedState, 500);
|
playbackRangeTimer = window.setInterval(publishBufferedState, 500);
|
||||||
}),
|
};
|
||||||
|
unsubscribers.push(
|
||||||
|
viewer.on("recording_open", admitRecording),
|
||||||
);
|
);
|
||||||
|
const discoverActiveLiveRecording = () => {
|
||||||
|
if (isRecordedSource || disposed || recordingOpened || !viewer.ready) return;
|
||||||
|
try {
|
||||||
|
const recordingId = viewer.get_active_recording_id();
|
||||||
|
if (!recordingId) return;
|
||||||
|
// Rerun 0.34.1 may ingest an SDK gRPC store without forwarding its
|
||||||
|
// recording_open event to the JavaScript wrapper. The active store
|
||||||
|
// is the authoritative fallback and avoids hiding a ready canvas.
|
||||||
|
postLiveViewerDiagnostic({
|
||||||
|
eventCode: "live_receiver_active_store_admitted",
|
||||||
|
streamId: liveStreamIdRef.current,
|
||||||
|
backendActivitySequence: liveActivitySequenceRef.current,
|
||||||
|
});
|
||||||
|
admitRecording({
|
||||||
|
application_id: "nodedc_mission_core_spatial",
|
||||||
|
recording_id: recordingId,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// The native receiver is still opening; the bounded timer retries.
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
unsubscribers.push(
|
unsubscribers.push(
|
||||||
viewer.on("time_update", (event) => {
|
viewer.on("time_update", (event) => {
|
||||||
@@ -1316,25 +1484,34 @@ export function RerunViewport({
|
|||||||
setPerceptionChannelRevision((revision) => revision + 1);
|
setPerceptionChannelRevision((revision) => revision + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!recordingOpened && !isRecordedSource) {
|
if (!isRecordedSource) {
|
||||||
|
discoverActiveLiveRecording();
|
||||||
|
if (!recordingOpened) {
|
||||||
|
liveRecordingDiscoveryTimer = window.setInterval(
|
||||||
|
discoverActiveLiveRecording,
|
||||||
|
100,
|
||||||
|
);
|
||||||
recordingOpenTimer = window.setTimeout(() => {
|
recordingOpenTimer = window.setTimeout(() => {
|
||||||
recordingOpenTimedOut = true;
|
recordingOpenTimedOut = true;
|
||||||
disposeViewer?.();
|
requestLiveRecovery("recording-open-timeout");
|
||||||
reportError("Визуализатор запущен, но поток записи не открылся.");
|
|
||||||
}, 12_000);
|
}, 12_000);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
disposeViewer();
|
disposeViewer();
|
||||||
if (!recordingOpenTimedOut && !disposed) {
|
if (!recordingOpenTimedOut && !disposed) {
|
||||||
reportError(isRecordedSource
|
const message = isRecordedSource
|
||||||
? "Запись не прошла атомарную проверку целостности."
|
? "Запись не прошла атомарную проверку целостности."
|
||||||
: "Не удалось запустить встроенный визуализатор.");
|
: "Не удалось запустить встроенный визуализатор.";
|
||||||
|
if (!isRecordedSource && requestLiveRecovery("viewer-start")) return;
|
||||||
|
reportError(message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
if (disposed) return;
|
if (disposed) return;
|
||||||
disposeViewer?.();
|
disposeViewer?.();
|
||||||
|
if (requestLiveRecovery("module-load")) return;
|
||||||
reportError("Не удалось загрузить модуль визуализатора.");
|
reportError("Не удалось загрузить модуль визуализатора.");
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -1355,6 +1532,7 @@ export function RerunViewport({
|
|||||||
expectedTimelineStartSeconds,
|
expectedTimelineStartSeconds,
|
||||||
followLive,
|
followLive,
|
||||||
initialPlaybackStartSeconds,
|
initialPlaybackStartSeconds,
|
||||||
|
liveStreamId,
|
||||||
onPlaybackChange,
|
onPlaybackChange,
|
||||||
onPlaybackControllerChange,
|
onPlaybackControllerChange,
|
||||||
onSelectionChange,
|
onSelectionChange,
|
||||||
|
|||||||
@@ -0,0 +1,156 @@
|
|||||||
|
export interface LiveReceiverWatchdogState {
|
||||||
|
lastBackendActivitySequence: number | null;
|
||||||
|
lastViewerRangeMaxNs: number | null;
|
||||||
|
staticRangeBackendAdvances: number;
|
||||||
|
stalledSinceMs: number | null;
|
||||||
|
stallReported: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type LiveReceiverWatchdogSignal = "none" | "receiver-advanced" | "stalled";
|
||||||
|
|
||||||
|
export interface LiveReceiverWatchdogResult {
|
||||||
|
state: LiveReceiverWatchdogState;
|
||||||
|
signal: LiveReceiverWatchdogSignal;
|
||||||
|
stalledForMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LiveReceiverRecoveryState {
|
||||||
|
attempts: number;
|
||||||
|
awaitingRecovery: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type LiveReceiverRecoverySignal = "retry" | "exhausted";
|
||||||
|
|
||||||
|
export interface LiveReceiverRecoveryResult {
|
||||||
|
state: LiveReceiverRecoveryState;
|
||||||
|
signal: LiveReceiverRecoverySignal;
|
||||||
|
attempt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const LIVE_RECEIVER_STALL_THRESHOLD_MS = 5_000;
|
||||||
|
export const LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS = 3;
|
||||||
|
|
||||||
|
export function initialLiveReceiverWatchdogState(): LiveReceiverWatchdogState {
|
||||||
|
return {
|
||||||
|
lastBackendActivitySequence: null,
|
||||||
|
lastViewerRangeMaxNs: null,
|
||||||
|
staticRangeBackendAdvances: 0,
|
||||||
|
stalledSinceMs: null,
|
||||||
|
stallReported: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function initialLiveReceiverRecoveryState(): LiveReceiverRecoveryState {
|
||||||
|
return {
|
||||||
|
attempts: 0,
|
||||||
|
awaitingRecovery: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bound viewer-only restarts independently from scanner and acquisition
|
||||||
|
* lifecycle. The caller may dispose and recreate the browser receiver, but
|
||||||
|
* must never issue START/STOP or reconnect the physical device.
|
||||||
|
*/
|
||||||
|
export function requestLiveReceiverRecovery(
|
||||||
|
current: LiveReceiverRecoveryState,
|
||||||
|
maxAttempts = LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
||||||
|
): LiveReceiverRecoveryResult {
|
||||||
|
if (current.attempts >= maxAttempts) {
|
||||||
|
return {
|
||||||
|
state: {
|
||||||
|
attempts: current.attempts,
|
||||||
|
awaitingRecovery: false,
|
||||||
|
},
|
||||||
|
signal: "exhausted",
|
||||||
|
attempt: current.attempts,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const attempt = current.attempts + 1;
|
||||||
|
return {
|
||||||
|
state: {
|
||||||
|
attempts: attempt,
|
||||||
|
awaitingRecovery: true,
|
||||||
|
},
|
||||||
|
signal: "retry",
|
||||||
|
attempt,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detect a disposable receiver stall only when the backend frame counter keeps
|
||||||
|
* advancing while Rerun's visible time range does not. Scanner commands are
|
||||||
|
* deliberately absent from this state machine.
|
||||||
|
*/
|
||||||
|
export function advanceLiveReceiverWatchdog(
|
||||||
|
current: LiveReceiverWatchdogState,
|
||||||
|
sample: {
|
||||||
|
nowMs: number;
|
||||||
|
backendActivitySequence: number | null;
|
||||||
|
viewerRangeMaxNs: number | null;
|
||||||
|
},
|
||||||
|
thresholdMs = LIVE_RECEIVER_STALL_THRESHOLD_MS,
|
||||||
|
): LiveReceiverWatchdogResult {
|
||||||
|
const backendSequence = Number.isSafeInteger(sample.backendActivitySequence) &&
|
||||||
|
(sample.backendActivitySequence ?? -1) >= 0
|
||||||
|
? sample.backendActivitySequence
|
||||||
|
: null;
|
||||||
|
const viewerRange = Number.isFinite(sample.viewerRangeMaxNs) &&
|
||||||
|
(sample.viewerRangeMaxNs ?? -1) >= 0
|
||||||
|
? sample.viewerRangeMaxNs
|
||||||
|
: null;
|
||||||
|
const state = { ...current };
|
||||||
|
|
||||||
|
if (state.lastBackendActivitySequence === null && state.lastViewerRangeMaxNs === null) {
|
||||||
|
state.lastBackendActivitySequence = backendSequence;
|
||||||
|
state.lastViewerRangeMaxNs = viewerRange;
|
||||||
|
return { state, signal: "none", stalledForMs: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
const backendAdvanced = backendSequence !== null &&
|
||||||
|
(state.lastBackendActivitySequence === null ||
|
||||||
|
backendSequence > state.lastBackendActivitySequence);
|
||||||
|
const receiverAdvanced = viewerRange !== null &&
|
||||||
|
(state.lastViewerRangeMaxNs === null || viewerRange > state.lastViewerRangeMaxNs);
|
||||||
|
|
||||||
|
if (backendSequence !== null) {
|
||||||
|
state.lastBackendActivitySequence = Math.max(
|
||||||
|
backendSequence,
|
||||||
|
state.lastBackendActivitySequence ?? 0,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (viewerRange !== null) {
|
||||||
|
state.lastViewerRangeMaxNs = Math.max(viewerRange, state.lastViewerRangeMaxNs ?? 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (receiverAdvanced) {
|
||||||
|
const hadStaticBackendProgress = state.stalledSinceMs !== null;
|
||||||
|
state.staticRangeBackendAdvances = 0;
|
||||||
|
state.stalledSinceMs = null;
|
||||||
|
state.stallReported = false;
|
||||||
|
return {
|
||||||
|
state,
|
||||||
|
signal: hadStaticBackendProgress ? "receiver-advanced" : "none",
|
||||||
|
stalledForMs: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (backendAdvanced) {
|
||||||
|
state.staticRangeBackendAdvances += 1;
|
||||||
|
if (state.staticRangeBackendAdvances >= 2 && state.stalledSinceMs === null) {
|
||||||
|
state.stalledSinceMs = sample.nowMs;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const stalledForMs = state.stalledSinceMs === null
|
||||||
|
? 0
|
||||||
|
: Math.max(0, sample.nowMs - state.stalledSinceMs);
|
||||||
|
if (
|
||||||
|
!state.stallReported &&
|
||||||
|
state.staticRangeBackendAdvances >= 2 &&
|
||||||
|
stalledForMs >= thresholdMs
|
||||||
|
) {
|
||||||
|
state.stallReported = true;
|
||||||
|
return { state, signal: "stalled", stalledForMs };
|
||||||
|
}
|
||||||
|
return { state, signal: "none", stalledForMs };
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
export type LiveViewerDiagnosticEventCode =
|
||||||
|
| "live_receiver_stalled"
|
||||||
|
| "live_receiver_restart_requested"
|
||||||
|
| "live_receiver_recovered"
|
||||||
|
| "live_receiver_recovery_exhausted"
|
||||||
|
| "live_receiver_active_store_admitted"
|
||||||
|
| "live_receiver_error";
|
||||||
|
|
||||||
|
export type LiveViewerFailureStage =
|
||||||
|
| "recording-open-timeout"
|
||||||
|
| "viewer-start"
|
||||||
|
| "module-load"
|
||||||
|
| "receiver-stalled";
|
||||||
|
|
||||||
|
export interface LiveViewerDiagnostic {
|
||||||
|
eventCode: LiveViewerDiagnosticEventCode;
|
||||||
|
failureStage?: LiveViewerFailureStage | null;
|
||||||
|
streamId?: string | null;
|
||||||
|
backendActivitySequence?: number | null;
|
||||||
|
viewerRangeMaxNs?: number | null;
|
||||||
|
stalledForMs?: number | null;
|
||||||
|
recoveryAttempt?: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const SAFE_STREAM_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||||
|
|
||||||
|
function safeInteger(value: number | null | undefined): number | undefined {
|
||||||
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0
|
||||||
|
? value
|
||||||
|
: undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function postLiveViewerDiagnostic(event: LiveViewerDiagnostic): void {
|
||||||
|
const streamId = event.streamId?.trim();
|
||||||
|
const body = {
|
||||||
|
schema_version: "missioncore.live-viewer-diagnostic/v1",
|
||||||
|
event_code: event.eventCode,
|
||||||
|
...(event.failureStage ? { failure_stage: event.failureStage } : {}),
|
||||||
|
...(streamId && SAFE_STREAM_ID.test(streamId) ? { stream_id: streamId } : {}),
|
||||||
|
...(safeInteger(event.backendActivitySequence) === undefined
|
||||||
|
? {}
|
||||||
|
: { backend_activity_sequence: safeInteger(event.backendActivitySequence) }),
|
||||||
|
...(safeInteger(event.viewerRangeMaxNs) === undefined
|
||||||
|
? {}
|
||||||
|
: { viewer_range_max_ns: safeInteger(event.viewerRangeMaxNs) }),
|
||||||
|
...(safeInteger(event.stalledForMs) === undefined
|
||||||
|
? {}
|
||||||
|
: { stalled_for_ms: safeInteger(event.stalledForMs) }),
|
||||||
|
...(safeInteger(event.recoveryAttempt) === undefined
|
||||||
|
? {}
|
||||||
|
: { recovery_attempt: safeInteger(event.recoveryAttempt) }),
|
||||||
|
};
|
||||||
|
void fetch("/api/v1/viewer/live-diagnostics", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
keepalive: true,
|
||||||
|
}).catch(() => {
|
||||||
|
// Diagnostics must never interfere with the live receiver recovery path.
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -28,6 +28,7 @@ export interface ViewerSettings {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface StreamMetrics {
|
export interface StreamMetrics {
|
||||||
|
publishedFrameCount?: number | null;
|
||||||
latencyMs?: number | null;
|
latencyMs?: number | null;
|
||||||
frameRateHz?: number | null;
|
frameRateHz?: number | null;
|
||||||
pointCount?: number | null;
|
pointCount?: number | null;
|
||||||
|
|||||||
@@ -499,7 +499,8 @@ function SpatialWorkspace({
|
|||||||
<RerunViewport
|
<RerunViewport
|
||||||
sourceUrl={sourceUrl}
|
sourceUrl={sourceUrl}
|
||||||
recordedArtifact={recordedSource ? recordedReplay : null}
|
recordedArtifact={recordedSource ? recordedReplay : null}
|
||||||
followLive={!recordedSource && state?.sourceMode === "live"}
|
followLive={!recordedReplay && streamActive}
|
||||||
|
liveActivitySequence={metrics?.publishedFrameCount} liveStreamId={state?.spatialSource?.id}
|
||||||
autoplayWhenReady={recordedSource}
|
autoplayWhenReady={recordedSource}
|
||||||
presentationGate={recordedSessionGate}
|
presentationGate={recordedSessionGate}
|
||||||
expectedTimelineStartSeconds={recordedSource
|
expectedTimelineStartSeconds={recordedSource
|
||||||
|
|||||||
@@ -439,6 +439,11 @@ test("live source is confirmed only by an acquiring acquisition", () => {
|
|||||||
assert.equal(lifecycle.sourceStatusLabel(waiting), "Ожидание реальных данных");
|
assert.equal(lifecycle.sourceStatusLabel(waiting), "Ожидание реальных данных");
|
||||||
assert.equal(lifecycle.isConfirmedLiveState(acquiring), true);
|
assert.equal(lifecycle.isConfirmedLiveState(acquiring), true);
|
||||||
assert.equal(lifecycle.confirmedRuntimeSourceMode(acquiring), "live");
|
assert.equal(lifecycle.confirmedRuntimeSourceMode(acquiring), "live");
|
||||||
|
assert.equal(lifecycle.liveStartPlan(acquiring), "already-running");
|
||||||
|
assert.equal(
|
||||||
|
lifecycle.controlSessionEntryPlan("scanning", false, false),
|
||||||
|
"continue",
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("prepared acquisition resumes without another prepare and remains recoverable", () => {
|
test("prepared acquisition resumes without another prepare and remains recoverable", () => {
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { after, before, test } from "node:test";
|
||||||
|
|
||||||
|
import { createServer } from "vite";
|
||||||
|
|
||||||
|
let server;
|
||||||
|
let advanceLiveReceiverWatchdog;
|
||||||
|
let initialLiveReceiverWatchdogState;
|
||||||
|
let initialLiveReceiverRecoveryState;
|
||||||
|
let requestLiveReceiverRecovery;
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
server = await createServer({
|
||||||
|
appType: "custom",
|
||||||
|
logLevel: "silent",
|
||||||
|
server: { middlewareMode: true },
|
||||||
|
});
|
||||||
|
({
|
||||||
|
advanceLiveReceiverWatchdog,
|
||||||
|
initialLiveReceiverRecoveryState,
|
||||||
|
initialLiveReceiverWatchdogState,
|
||||||
|
requestLiveReceiverRecovery,
|
||||||
|
} = await server.ssrLoadModule("/src/core/observation/liveReceiverWatchdog.ts"));
|
||||||
|
});
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
await server?.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("watchdog ignores healthy live progress", () => {
|
||||||
|
let state = initialLiveReceiverWatchdogState();
|
||||||
|
for (let index = 0; index < 20; index += 1) {
|
||||||
|
const observed = advanceLiveReceiverWatchdog(state, {
|
||||||
|
nowMs: index * 500,
|
||||||
|
backendActivitySequence: index * 2,
|
||||||
|
viewerRangeMaxNs: index * 1_000_000_000,
|
||||||
|
});
|
||||||
|
state = observed.state;
|
||||||
|
assert.notEqual(observed.signal, "stalled");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("watchdog detects a frozen receiver while backend publication advances", () => {
|
||||||
|
let state = initialLiveReceiverWatchdogState();
|
||||||
|
let signal = "none";
|
||||||
|
let stalledForMs = 0;
|
||||||
|
for (let index = 0; index <= 13; index += 1) {
|
||||||
|
const observed = advanceLiveReceiverWatchdog(state, {
|
||||||
|
nowMs: index * 500,
|
||||||
|
backendActivitySequence: 100 + index,
|
||||||
|
viewerRangeMaxNs: 42_000_000_000,
|
||||||
|
});
|
||||||
|
state = observed.state;
|
||||||
|
signal = observed.signal === "stalled" ? observed.signal : signal;
|
||||||
|
stalledForMs = Math.max(stalledForMs, observed.stalledForMs);
|
||||||
|
}
|
||||||
|
assert.equal(signal, "stalled");
|
||||||
|
assert.ok(stalledForMs >= 5_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("one backend increment at stream finalization does not create a false stall", () => {
|
||||||
|
let state = initialLiveReceiverWatchdogState();
|
||||||
|
state = advanceLiveReceiverWatchdog(state, {
|
||||||
|
nowMs: 0,
|
||||||
|
backendActivitySequence: 10,
|
||||||
|
viewerRangeMaxNs: 20_000_000_000,
|
||||||
|
}).state;
|
||||||
|
state = advanceLiveReceiverWatchdog(state, {
|
||||||
|
nowMs: 500,
|
||||||
|
backendActivitySequence: 11,
|
||||||
|
viewerRangeMaxNs: 20_000_000_000,
|
||||||
|
}).state;
|
||||||
|
const observed = advanceLiveReceiverWatchdog(state, {
|
||||||
|
nowMs: 10_000,
|
||||||
|
backendActivitySequence: 11,
|
||||||
|
viewerRangeMaxNs: 20_000_000_000,
|
||||||
|
});
|
||||||
|
assert.equal(observed.signal, "none");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("startup failures request only three bounded viewer restarts", () => {
|
||||||
|
let state = initialLiveReceiverRecoveryState();
|
||||||
|
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
||||||
|
const recovery = requestLiveReceiverRecovery(state);
|
||||||
|
assert.equal(recovery.signal, "retry");
|
||||||
|
assert.equal(recovery.attempt, attempt);
|
||||||
|
assert.equal(recovery.state.awaitingRecovery, true);
|
||||||
|
state = recovery.state;
|
||||||
|
}
|
||||||
|
const exhausted = requestLiveReceiverRecovery(state);
|
||||||
|
assert.equal(exhausted.signal, "exhausted");
|
||||||
|
assert.equal(exhausted.attempt, 3);
|
||||||
|
assert.equal(exhausted.state.awaitingRecovery, false);
|
||||||
|
});
|
||||||
@@ -129,7 +129,11 @@ test("recorded RRD bytes are never split across LogChannel.send_rrd calls", asyn
|
|||||||
assert.match(source, /viewer\.start\(\s*rerunViewerInitialSource\(resolvedSource\)/s);
|
assert.match(source, /viewer\.start\(\s*rerunViewerInitialSource\(resolvedSource\)/s);
|
||||||
assert.match(
|
assert.match(
|
||||||
source,
|
source,
|
||||||
/recordingOpened = true;[\s\S]*if \(!isRecordedSource\) clearLiveRecordingOpenTimer\(\);/,
|
/recordingOpened = true;[\s\S]*clearLiveRecordingOpenTimer\(\);[\s\S]*clearLiveRecordingDiscoveryTimer\(\);/,
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
source,
|
||||||
|
/viewer\.get_active_recording_id\(\)[\s\S]*eventCode: "live_receiver_active_store_admitted"[\s\S]*admitRecording\(\{[\s\S]*application_id: "nodedc_mission_core_spatial"/,
|
||||||
);
|
);
|
||||||
assert.match(
|
assert.match(
|
||||||
source,
|
source,
|
||||||
@@ -141,6 +145,14 @@ test("recorded RRD bytes are never split across LogChannel.send_rrd calls", asyn
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("raw replay exercises the same streaming receiver lifecycle as a live scan", async () => {
|
||||||
|
const source = await readFile(
|
||||||
|
new URL("../src/workspaces/Workspaces.tsx", import.meta.url),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
assert.match(source, /followLive=\{!recordedReplay && streamActive\}/);
|
||||||
|
});
|
||||||
|
|
||||||
test("the complete vendor canvas host is hidden during partial and failed admission", async () => {
|
test("the complete vendor canvas host is hidden during partial and failed admission", async () => {
|
||||||
const css = await readFile(new URL("../src/styles/spatial.css", import.meta.url), "utf8");
|
const css = await readFile(new URL("../src/styles/spatial.css", import.meta.url), "utf8");
|
||||||
assert.match(
|
assert.match(
|
||||||
|
|||||||
@@ -58,6 +58,30 @@ export interface XgridsDeviceSession {
|
|||||||
connectivity?: "unknown" | "offline" | "connecting" | "connected" | "degraded";
|
connectivity?: "unknown" | "offline" | "connecting" | "connected" | "degraded";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface XgridsConnectionVerification {
|
||||||
|
status?:
|
||||||
|
| "not-probed"
|
||||||
|
| "configured"
|
||||||
|
| "live-address-observed"
|
||||||
|
| "reachable"
|
||||||
|
| "recovered"
|
||||||
|
| "unreachable";
|
||||||
|
lease_state?: "disconnected" | "configured" | "reachable";
|
||||||
|
lease_generation?: number;
|
||||||
|
endpoint_validation?: string | null;
|
||||||
|
network_reachability?:
|
||||||
|
| "unknown"
|
||||||
|
| "not-probed"
|
||||||
|
| "reachable"
|
||||||
|
| "degraded"
|
||||||
|
| "unreachable";
|
||||||
|
address_changed?: boolean;
|
||||||
|
previous_address_present?: boolean;
|
||||||
|
write_performed?: boolean;
|
||||||
|
observed_at?: string | null;
|
||||||
|
reason_code?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
export interface XgridsCompatibilityState {
|
export interface XgridsCompatibilityState {
|
||||||
profile_id?: string | null;
|
profile_id?: string | null;
|
||||||
decision?: "compatible" | "limited" | "unknown" | "incompatible";
|
decision?: "compatible" | "limited" | "unknown" | "incompatible";
|
||||||
@@ -167,6 +191,23 @@ export interface XgridsApplicationControlSession {
|
|||||||
expected?: Record<string, unknown>;
|
expected?: Record<string, unknown>;
|
||||||
observed?: Record<string, unknown>;
|
observed?: Record<string, unknown>;
|
||||||
} | null;
|
} | null;
|
||||||
|
status_reconciliation?: {
|
||||||
|
device_session_state?: string;
|
||||||
|
device_project_bound?: boolean;
|
||||||
|
system_error_code?: number | null;
|
||||||
|
decision?: "safe-explicit-prestart-retry";
|
||||||
|
automatic_retry?: false;
|
||||||
|
} | null;
|
||||||
|
network_change_admissible?: boolean;
|
||||||
|
network_change_reconciliation?: {
|
||||||
|
device_session_state?: "scan_stopping";
|
||||||
|
device_project_bound?: true;
|
||||||
|
system_error_code?: null;
|
||||||
|
stop_complete?: true;
|
||||||
|
standby_confirmed?: false;
|
||||||
|
decision?: "explicit-network-change-only-after-acknowledged-stop";
|
||||||
|
automatic_retry?: false;
|
||||||
|
} | null;
|
||||||
safe_to_retry?: boolean;
|
safe_to_retry?: boolean;
|
||||||
} | null;
|
} | null;
|
||||||
dialogue?: Record<string, unknown> | null;
|
dialogue?: Record<string, unknown> | null;
|
||||||
@@ -220,6 +261,8 @@ export interface XgridsOperation {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface XgridsK1Metrics {
|
export interface XgridsK1Metrics {
|
||||||
|
pcl_frames?: number | null;
|
||||||
|
pose_frames?: number | null;
|
||||||
mqtt_to_decode_ms?: number | null;
|
mqtt_to_decode_ms?: number | null;
|
||||||
decode_ms?: number | null;
|
decode_ms?: number | null;
|
||||||
publish_ms?: number | null;
|
publish_ms?: number | null;
|
||||||
@@ -308,6 +351,7 @@ export interface XgridsK1State {
|
|||||||
application_control_session?: XgridsApplicationControlSession | null;
|
application_control_session?: XgridsApplicationControlSession | null;
|
||||||
device_ref?: XgridsDeviceRef | null;
|
device_ref?: XgridsDeviceRef | null;
|
||||||
device_session?: XgridsDeviceSession | null;
|
device_session?: XgridsDeviceSession | null;
|
||||||
|
connection_verification?: XgridsConnectionVerification | null;
|
||||||
acquisition?: XgridsAcquisition | null;
|
acquisition?: XgridsAcquisition | null;
|
||||||
operations?: XgridsOperation[];
|
operations?: XgridsOperation[];
|
||||||
last_operation?: XgridsOperation | null;
|
last_operation?: XgridsOperation | null;
|
||||||
|
|||||||
@@ -104,6 +104,14 @@ function normalizeState(
|
|||||||
viewerSettings: state.viewer_settings,
|
viewerSettings: state.viewer_settings,
|
||||||
sourceMode: confirmedRuntimeSourceMode(state),
|
sourceMode: confirmedRuntimeSourceMode(state),
|
||||||
metrics: {
|
metrics: {
|
||||||
|
publishedFrameCount: (
|
||||||
|
Number.isSafeInteger(metrics?.pcl_frames) &&
|
||||||
|
Number.isSafeInteger(metrics?.pose_frames) &&
|
||||||
|
(metrics?.pcl_frames ?? -1) >= 0 &&
|
||||||
|
(metrics?.pose_frames ?? -1) >= 0
|
||||||
|
)
|
||||||
|
? (metrics?.pcl_frames ?? 0) + (metrics?.pose_frames ?? 0)
|
||||||
|
: null,
|
||||||
latencyMs: pipelineLatency(metrics),
|
latencyMs: pipelineLatency(metrics),
|
||||||
frameRateHz: finiteMetric(metrics?.frame_rate ?? metrics?.frame_rate_hz),
|
frameRateHz: finiteMetric(metrics?.frame_rate ?? metrics?.frame_rate_hz),
|
||||||
pointCount: finiteMetric(metrics?.point_count),
|
pointCount: finiteMetric(metrics?.point_count),
|
||||||
|
|||||||
@@ -70,6 +70,8 @@ function controlFailure(state: XgridsK1State): ApiError {
|
|||||||
"Сканер не подтвердил канонические MQTT-подписки.",
|
"Сканер не подтвердил канонические MQTT-подписки.",
|
||||||
mqtt_response_timeout:
|
mqtt_response_timeout:
|
||||||
"Ожидаемый ответ сканера не пришёл до безопасной границы ожидания.",
|
"Ожидаемый ответ сканера не пришёл до безопасной границы ожидания.",
|
||||||
|
mqtt_network_loop_failed:
|
||||||
|
"Локальный MQTT-клиент потерял управляющее соединение со сканером.",
|
||||||
response_identity_decode_failed:
|
response_identity_decode_failed:
|
||||||
"Ответ сканера не удалось безопасно разобрать и привязать к операции.",
|
"Ответ сканера не удалось безопасно разобрать и привязать к операции.",
|
||||||
modeling_response_decode_failed:
|
modeling_response_decode_failed:
|
||||||
@@ -124,7 +126,9 @@ function controlFailure(state: XgridsK1State): ApiError {
|
|||||||
? "Команды START и STOP не отправлялись."
|
? "Команды START и STOP не отправлялись."
|
||||||
: "Факт отправки START или STOP диагностически не подтверждён; повтор запрещён.";
|
: "Факт отправки START или STOP диагностически не подтверждён; повтор запрещён.";
|
||||||
const retryStatus = failure?.safe_to_retry
|
const retryStatus = failure?.safe_to_retry
|
||||||
? "Новая попытка возможна только отдельным нажатием оператора."
|
? failure.status_reconciliation?.decision === "safe-explicit-prestart-retry"
|
||||||
|
? "Живой статус READY без связанного проекта подтверждён; новая попытка возможна только отдельным нажатием оператора."
|
||||||
|
: "Новая попытка возможна только отдельным нажатием оператора."
|
||||||
: "Повтор заблокирован до ручной проверки состояния.";
|
: "Повтор заблокирован до ручной проверки состояния.";
|
||||||
const exchanges = typeof failure?.publish_attempts === "number"
|
const exchanges = typeof failure?.publish_attempts === "number"
|
||||||
? `MQTT-публикаций до остановки: ${failure.publish_attempts}.`
|
? `MQTT-публикаций до остановки: ${failure.publish_attempts}.`
|
||||||
|
|||||||
@@ -396,6 +396,11 @@ class LivePerceptionIngress:
|
|||||||
if self._session_id == session_id:
|
if self._session_id == session_id:
|
||||||
return
|
return
|
||||||
raise RuntimeError("another live perception session is active")
|
raise RuntimeError("another live perception session is active")
|
||||||
|
# Queued items belong to one acquisition. A worker may be absent
|
||||||
|
# while a session ends, so clear bounded leftovers before the next
|
||||||
|
# session can become visible to that worker.
|
||||||
|
for queue in self._queues.values():
|
||||||
|
queue.items.clear()
|
||||||
self._session_id = session_id
|
self._session_id = session_id
|
||||||
self._active = True
|
self._active = True
|
||||||
self._publish_locked(
|
self._publish_locked(
|
||||||
@@ -681,9 +686,7 @@ class LiveSensorSynchronizer:
|
|||||||
or retention_seconds <= 0
|
or retention_seconds <= 0
|
||||||
):
|
):
|
||||||
raise ValueError("live sensor synchronizer bounds are invalid")
|
raise ValueError("live sensor synchronizer bounds are invalid")
|
||||||
self._maximum_lidar_camera_delta_ns = round(
|
self._maximum_lidar_camera_delta_ns = round(maximum_lidar_camera_delta_ms * 1_000_000)
|
||||||
maximum_lidar_camera_delta_ms * 1_000_000
|
|
||||||
)
|
|
||||||
self._maximum_pose_point_delta_ns = round(maximum_pose_point_delta_ms * 1_000_000)
|
self._maximum_pose_point_delta_ns = round(maximum_pose_point_delta_ms * 1_000_000)
|
||||||
self._capacity = capacity_per_modality
|
self._capacity = capacity_per_modality
|
||||||
self._retention_ns = round(retention_seconds * 1_000_000_000)
|
self._retention_ns = round(retention_seconds * 1_000_000_000)
|
||||||
@@ -801,8 +804,7 @@ class LiveSensorSynchronizer:
|
|||||||
newest = values[-1].context.captured_at_epoch_ns
|
newest = values[-1].context.captured_at_epoch_ns
|
||||||
oldest_allowed = newest - self._retention_ns
|
oldest_allowed = newest - self._retention_ns
|
||||||
while values and (
|
while values and (
|
||||||
len(values) > self._capacity
|
len(values) > self._capacity or values[0].context.captured_at_epoch_ns < oldest_allowed
|
||||||
or values[0].context.captured_at_epoch_ns < oldest_allowed
|
|
||||||
):
|
):
|
||||||
values.popleft()
|
values.popleft()
|
||||||
removed += 1
|
removed += 1
|
||||||
@@ -837,9 +839,7 @@ class WorldStateProjector:
|
|||||||
if velocity_history_limit_s <= 0:
|
if velocity_history_limit_s <= 0:
|
||||||
raise ValueError("velocity history limit must be positive")
|
raise ValueError("velocity history limit must be positive")
|
||||||
self._velocity_history_limit_s = velocity_history_limit_s
|
self._velocity_history_limit_s = velocity_history_limit_s
|
||||||
self._track_history: dict[
|
self._track_history: dict[int, deque[tuple[float, tuple[float, float, float]]]] = {}
|
||||||
int, deque[tuple[float, tuple[float, float, float]]]
|
|
||||||
] = {}
|
|
||||||
|
|
||||||
def project(
|
def project(
|
||||||
self,
|
self,
|
||||||
@@ -950,9 +950,7 @@ class WorldStateProjector:
|
|||||||
math.sqrt(
|
math.sqrt(
|
||||||
sum(
|
sum(
|
||||||
(observed - expected) ** 2
|
(observed - expected) ** 2
|
||||||
for observed, expected in zip(
|
for observed, expected in zip(observed_center, predicted, strict=True)
|
||||||
observed_center, predicted, strict=True
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -163,13 +163,18 @@ async def read_wifi_status_once(
|
|||||||
device_macos_uuid: str,
|
device_macos_uuid: str,
|
||||||
*,
|
*,
|
||||||
timeout_seconds: float = 20.0,
|
timeout_seconds: float = 20.0,
|
||||||
|
rediscover: bool = False,
|
||||||
) -> WifiStatusReadResult:
|
) -> WifiStatusReadResult:
|
||||||
"""Read the K1's current DHCP status over BLE without writing a characteristic."""
|
"""Read the K1's current DHCP status over BLE without writing a characteristic."""
|
||||||
|
|
||||||
if timeout_seconds <= 0:
|
if timeout_seconds <= 0:
|
||||||
raise ValueError("timeout_seconds must be positive")
|
raise ValueError("timeout_seconds must be positive")
|
||||||
async with asyncio.timeout(timeout_seconds + 5.0):
|
async with asyncio.timeout(timeout_seconds + 5.0):
|
||||||
device = discovered_device(device_macos_uuid)
|
# A CoreBluetooth handle retained by an earlier scan is an optimization,
|
||||||
|
# not durable connection state. Recovery after sleep, Wi-Fi transition,
|
||||||
|
# or a completed provisioning GATT session must rediscover the device
|
||||||
|
# instead of repeatedly opening a stale handle.
|
||||||
|
device = None if rediscover else discovered_device(device_macos_uuid)
|
||||||
if device is None:
|
if device is None:
|
||||||
device = await BleakScanner.find_device_by_address(
|
device = await BleakScanner.find_device_by_address(
|
||||||
device_macos_uuid,
|
device_macos_uuid,
|
||||||
@@ -183,9 +188,7 @@ async def read_wifi_status_once(
|
|||||||
|
|
||||||
async with BleakClient(device, timeout=timeout_seconds, pair=False) as client:
|
async with BleakClient(device, timeout=timeout_seconds, pair=False) as client:
|
||||||
service = client.services.get_service(SERVICE_UUID)
|
service = client.services.get_service(SERVICE_UUID)
|
||||||
status_characteristic = client.services.get_characteristic(
|
status_characteristic = client.services.get_characteristic(STATUS_CHARACTERISTIC_UUID)
|
||||||
STATUS_CHARACTERISTIC_UUID
|
|
||||||
)
|
|
||||||
if service is None:
|
if service is None:
|
||||||
raise ValueError(f"Reviewed K1 service not found: {SERVICE_UUID}")
|
raise ValueError(f"Reviewed K1 service not found: {SERVICE_UUID}")
|
||||||
if status_characteristic is None:
|
if status_characteristic is None:
|
||||||
@@ -193,9 +196,7 @@ async def read_wifi_status_once(
|
|||||||
f"Reviewed K1 status characteristic not found: {STATUS_CHARACTERISTIC_UUID}"
|
f"Reviewed K1 status characteristic not found: {STATUS_CHARACTERISTIC_UUID}"
|
||||||
)
|
)
|
||||||
if status_characteristic.service_uuid != service.uuid:
|
if status_characteristic.service_uuid != service.uuid:
|
||||||
raise ValueError(
|
raise ValueError("K1 status characteristic is attached to an unexpected service")
|
||||||
"K1 status characteristic is attached to an unexpected service"
|
|
||||||
)
|
|
||||||
if "read" not in status_characteristic.properties:
|
if "read" not in status_characteristic.properties:
|
||||||
raise ValueError("Reviewed K1 status characteristic is not readable")
|
raise ValueError("Reviewed K1 status characteristic is not readable")
|
||||||
value = bytes(await client.read_gatt_char(status_characteristic))
|
value = bytes(await client.read_gatt_char(status_characteristic))
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -243,6 +243,9 @@ class ApplicationMqttTransportSnapshot:
|
|||||||
latest_device_init_ready: bool | None
|
latest_device_init_ready: bool | None
|
||||||
latest_system_error_code: int | None
|
latest_system_error_code: int | None
|
||||||
latest_system_error_state: str | None
|
latest_system_error_state: str | None
|
||||||
|
last_loop_result_code: int | None
|
||||||
|
last_loop_result_name: str | None
|
||||||
|
last_loop_phase: str | None
|
||||||
operation_keys_consumed: int
|
operation_keys_consumed: int
|
||||||
clean_session: bool = False
|
clean_session: bool = False
|
||||||
keepalive_seconds: int = CONTROL_KEEPALIVE_SECONDS
|
keepalive_seconds: int = CONTROL_KEEPALIVE_SECONDS
|
||||||
@@ -268,6 +271,9 @@ class ApplicationMqttTransportSnapshot:
|
|||||||
"latest_device_init_ready": self.latest_device_init_ready,
|
"latest_device_init_ready": self.latest_device_init_ready,
|
||||||
"latest_system_error_code": self.latest_system_error_code,
|
"latest_system_error_code": self.latest_system_error_code,
|
||||||
"latest_system_error_state": self.latest_system_error_state,
|
"latest_system_error_state": self.latest_system_error_state,
|
||||||
|
"last_loop_result_code": self.last_loop_result_code,
|
||||||
|
"last_loop_result_name": self.last_loop_result_name,
|
||||||
|
"last_loop_phase": self.last_loop_phase,
|
||||||
"operation_keys_consumed": self.operation_keys_consumed,
|
"operation_keys_consumed": self.operation_keys_consumed,
|
||||||
"clean_session": self.clean_session,
|
"clean_session": self.clean_session,
|
||||||
"keepalive_seconds": self.keepalive_seconds,
|
"keepalive_seconds": self.keepalive_seconds,
|
||||||
@@ -348,6 +354,9 @@ class ReviewedApplicationMqttTransport:
|
|||||||
self._latest_device_fault = False
|
self._latest_device_fault = False
|
||||||
self._latest_system_error_code: int | None = None
|
self._latest_system_error_code: int | None = None
|
||||||
self._latest_system_error_state: str | None = None
|
self._latest_system_error_state: str | None = None
|
||||||
|
self._last_loop_result_code: int | None = None
|
||||||
|
self._last_loop_result_name: str | None = None
|
||||||
|
self._last_loop_phase: str | None = None
|
||||||
|
|
||||||
def open(self) -> ApplicationMqttTransportSnapshot:
|
def open(self) -> ApplicationMqttTransportSnapshot:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
@@ -562,7 +571,9 @@ class ReviewedApplicationMqttTransport:
|
|||||||
except (OSError, RuntimeError, ValueError) as exc:
|
except (OSError, RuntimeError, ValueError) as exc:
|
||||||
self._fail_after_publish("control MQTT network loop failed", exc)
|
self._fail_after_publish("control MQTT network loop failed", exc)
|
||||||
if result != mqtt.MQTT_ERR_SUCCESS:
|
if result != mqtt.MQTT_ERR_SUCCESS:
|
||||||
self._fail_after_publish("control MQTT network loop returned an error")
|
self._fail_after_publish(
|
||||||
|
self._record_loop_failure(result, phase="maintain-open")
|
||||||
|
)
|
||||||
self._discard_allowed_responses(allowed)
|
self._discard_allowed_responses(allowed)
|
||||||
|
|
||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
@@ -649,6 +660,9 @@ class ReviewedApplicationMqttTransport:
|
|||||||
latest_device_init_ready=self._latest_device_init_ready,
|
latest_device_init_ready=self._latest_device_init_ready,
|
||||||
latest_system_error_code=self._latest_system_error_code,
|
latest_system_error_code=self._latest_system_error_code,
|
||||||
latest_system_error_state=self._latest_system_error_state,
|
latest_system_error_state=self._latest_system_error_state,
|
||||||
|
last_loop_result_code=self._last_loop_result_code,
|
||||||
|
last_loop_result_name=self._last_loop_result_name,
|
||||||
|
last_loop_phase=self._last_loop_phase,
|
||||||
operation_keys_consumed=len(self._consumed_operation_keys),
|
operation_keys_consumed=len(self._consumed_operation_keys),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -879,9 +893,13 @@ class ReviewedApplicationMqttTransport:
|
|||||||
self._fail_after_publish("control MQTT network loop failed", exc)
|
self._fail_after_publish("control MQTT network loop failed", exc)
|
||||||
self._fail_before_publish("control MQTT network loop failed", exc)
|
self._fail_before_publish("control MQTT network loop failed", exc)
|
||||||
if result != mqtt.MQTT_ERR_SUCCESS:
|
if result != mqtt.MQTT_ERR_SUCCESS:
|
||||||
|
failure = self._record_loop_failure(
|
||||||
|
result,
|
||||||
|
phase="post-publish" if post_publish else "connect-subscribe",
|
||||||
|
)
|
||||||
if post_publish:
|
if post_publish:
|
||||||
self._fail_after_publish("control MQTT network loop returned an error")
|
self._fail_after_publish(failure)
|
||||||
self._fail_before_publish("control MQTT network loop returned an error")
|
self._fail_before_publish(failure)
|
||||||
|
|
||||||
def _service_once(self, *, post_publish: bool) -> None:
|
def _service_once(self, *, post_publish: bool) -> None:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
@@ -898,9 +916,33 @@ class ReviewedApplicationMqttTransport:
|
|||||||
self._fail_after_publish("control MQTT network loop failed", exc)
|
self._fail_after_publish("control MQTT network loop failed", exc)
|
||||||
self._fail_before_publish("control MQTT network loop failed", exc)
|
self._fail_before_publish("control MQTT network loop failed", exc)
|
||||||
if result != mqtt.MQTT_ERR_SUCCESS:
|
if result != mqtt.MQTT_ERR_SUCCESS:
|
||||||
|
failure = self._record_loop_failure(
|
||||||
|
result,
|
||||||
|
phase="post-publish-drain" if post_publish else "pre-publish-drain",
|
||||||
|
)
|
||||||
if post_publish:
|
if post_publish:
|
||||||
self._fail_after_publish("control MQTT network loop returned an error")
|
self._fail_after_publish(failure)
|
||||||
self._fail_before_publish("control MQTT network loop returned an error")
|
self._fail_before_publish(failure)
|
||||||
|
|
||||||
|
def _record_loop_failure(
|
||||||
|
self,
|
||||||
|
result: int,
|
||||||
|
*,
|
||||||
|
phase: str,
|
||||||
|
) -> str:
|
||||||
|
result_code = int(result)
|
||||||
|
try:
|
||||||
|
result_name = mqtt.error_string(result_code)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
result_name = "unknown MQTT error"
|
||||||
|
with self._lock:
|
||||||
|
self._last_loop_result_code = result_code
|
||||||
|
self._last_loop_result_name = result_name
|
||||||
|
self._last_loop_phase = phase
|
||||||
|
return (
|
||||||
|
"control MQTT network loop returned an error "
|
||||||
|
f"(phase={phase}, result={result_code}: {result_name})"
|
||||||
|
)
|
||||||
|
|
||||||
def _drain_responses(
|
def _drain_responses(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -62,6 +62,7 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
|
|
||||||
TransportFactory = Callable[[str], ReviewedApplicationMqttTransport]
|
TransportFactory = Callable[[str], ReviewedApplicationMqttTransport]
|
||||||
|
ScanningObserver = Callable[[], None]
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -97,10 +98,13 @@ class InteractiveApplicationControlSession:
|
|||||||
*,
|
*,
|
||||||
transport_factory: TransportFactory = ReviewedApplicationMqttTransport,
|
transport_factory: TransportFactory = ReviewedApplicationMqttTransport,
|
||||||
epoch_seconds: Callable[[], int] = lambda: int(time.time()),
|
epoch_seconds: Callable[[], int] = lambda: int(time.time()),
|
||||||
|
scanning_observer: ScanningObserver | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._authority_loader = authority_loader
|
self._authority_loader = authority_loader
|
||||||
self._transport_factory = transport_factory
|
self._transport_factory = transport_factory
|
||||||
self._epoch_seconds = epoch_seconds
|
self._epoch_seconds = epoch_seconds
|
||||||
|
self._scanning_observer = scanning_observer
|
||||||
|
self._scanning_observer_errors = 0
|
||||||
self._lock = threading.RLock()
|
self._lock = threading.RLock()
|
||||||
self._phase: ApplicationControlPhase = "idle"
|
self._phase: ApplicationControlPhase = "idle"
|
||||||
self._host: str | None = None
|
self._host: str | None = None
|
||||||
@@ -215,6 +219,55 @@ class InteractiveApplicationControlSession:
|
|||||||
transport.close()
|
transport.close()
|
||||||
return self.snapshot()
|
return self.snapshot()
|
||||||
|
|
||||||
|
def retire_for_network_change(self) -> dict[str, object]:
|
||||||
|
"""Retire local control ownership for one new explicit network action.
|
||||||
|
|
||||||
|
A correlated STOP followed by a network-loop loss in SCAN_STOPPING is
|
||||||
|
not enough evidence to authorize another START. It is enough to retire
|
||||||
|
the dead local socket when the operator explicitly chooses a new
|
||||||
|
network route. Pre-START failures already classified safe for a fresh
|
||||||
|
click are admissible here for the same reason: no modeling command was
|
||||||
|
attempted.
|
||||||
|
"""
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
failure = self._failure
|
||||||
|
failed_network_change_admissible = (
|
||||||
|
self._phase == "failed"
|
||||||
|
and failure is not None
|
||||||
|
and (
|
||||||
|
failure.get("safe_to_retry") is True
|
||||||
|
or failure.get("network_change_admissible") is True
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if self._phase not in {"idle", "completed", "closed"} and not (
|
||||||
|
failed_network_change_admissible
|
||||||
|
):
|
||||||
|
raise ApplicationAcceptanceError(
|
||||||
|
"control session cannot be retired for a network change"
|
||||||
|
)
|
||||||
|
if not self._worker_retired_locked():
|
||||||
|
raise ApplicationAcceptanceError(
|
||||||
|
"control session worker is still retiring"
|
||||||
|
)
|
||||||
|
previous_phase = self._phase
|
||||||
|
previous_reason = (
|
||||||
|
self._json_string(failure.get("reason_code"))
|
||||||
|
if failure is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
self._reset_locked()
|
||||||
|
if previous_phase != "idle":
|
||||||
|
logger.info(
|
||||||
|
"K1 local control session retired for an explicit network change",
|
||||||
|
extra={
|
||||||
|
"event_code": "k1_control_session_retired_for_network_change",
|
||||||
|
"reason_code": previous_reason,
|
||||||
|
"automatic_retry": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return self.snapshot()
|
||||||
|
|
||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
"""Stop the process-owned socket without ever inventing a device STOP."""
|
"""Stop the process-owned socket without ever inventing a device STOP."""
|
||||||
|
|
||||||
@@ -247,6 +300,7 @@ class InteractiveApplicationControlSession:
|
|||||||
"scripted_transitions": False,
|
"scripted_transitions": False,
|
||||||
"automatic_retry": False,
|
"automatic_retry": False,
|
||||||
"outcome_unknown": self._outcome_unknown,
|
"outcome_unknown": self._outcome_unknown,
|
||||||
|
"scanning_observer_errors": self._scanning_observer_errors,
|
||||||
"failure": dict(self._failure) if self._failure is not None else None,
|
"failure": dict(self._failure) if self._failure is not None else None,
|
||||||
"dialogue": (
|
"dialogue": (
|
||||||
dict(self._dialogue_snapshot) if self._dialogue_snapshot is not None else None
|
dict(self._dialogue_snapshot) if self._dialogue_snapshot is not None else None
|
||||||
@@ -310,6 +364,7 @@ class InteractiveApplicationControlSession:
|
|||||||
checkpoint=start_checkpoint,
|
checkpoint=start_checkpoint,
|
||||||
)
|
)
|
||||||
self._set_phase("scanning")
|
self._set_phase("scanning")
|
||||||
|
self._notify_scanning_observer()
|
||||||
|
|
||||||
executor.maintain_active_until_stop_requested(self._stop_requested.is_set)
|
executor.maintain_active_until_stop_requested(self._stop_requested.is_set)
|
||||||
stop_confirmation = self._stop_request()
|
stop_confirmation = self._stop_request()
|
||||||
@@ -412,11 +467,53 @@ class InteractiveApplicationControlSession:
|
|||||||
)
|
)
|
||||||
== 1
|
== 1
|
||||||
)
|
)
|
||||||
safe_to_retry = not outcome_unknown and (
|
status_reconciled_prestart_failure = (
|
||||||
|
outcome_unknown
|
||||||
|
and modeling_command_attempted is False
|
||||||
|
and transport_snapshot_available
|
||||||
|
and not diagnostic_evidence_unavailable
|
||||||
|
and self._json_int_or_none(
|
||||||
|
transport_snapshot.get("device_status_reports")
|
||||||
|
)
|
||||||
|
not in {None, 0}
|
||||||
|
and self._json_string(
|
||||||
|
transport_snapshot.get("latest_device_session_state")
|
||||||
|
)
|
||||||
|
== "ready"
|
||||||
|
and transport_snapshot.get("latest_device_project_bound") is False
|
||||||
|
and self._json_int_or_none(
|
||||||
|
transport_snapshot.get("latest_system_error_code")
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
stop_acknowledged_network_change = (
|
||||||
|
outcome_unknown
|
||||||
|
and failure_reason_code == "mqtt_network_loop_failed"
|
||||||
|
and failed_phase == "awaiting-standby-confirmation"
|
||||||
|
and dialogue_snapshot_available
|
||||||
|
and transport_snapshot_available
|
||||||
|
and dialogue_snapshot.get("stop_attempted") is True
|
||||||
|
and dialogue_snapshot.get("stop_complete") is True
|
||||||
|
and dialogue_stage == "stop-acknowledged"
|
||||||
|
and modeling_command_attempted is True
|
||||||
|
and self._json_string(
|
||||||
|
transport_snapshot.get("latest_device_session_state")
|
||||||
|
)
|
||||||
|
== "scan_stopping"
|
||||||
|
and transport_snapshot.get("latest_device_project_bound") is True
|
||||||
|
and self._json_int_or_none(
|
||||||
|
transport_snapshot.get("latest_system_error_code")
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
safe_to_retry = status_reconciled_prestart_failure or (
|
||||||
|
not outcome_unknown
|
||||||
|
and (
|
||||||
not transport_created
|
not transport_created
|
||||||
or (transport_snapshot_available and publish_attempts == 0)
|
or (transport_snapshot_available and publish_attempts == 0)
|
||||||
or correlated_read_only_profile_mismatch
|
or correlated_read_only_profile_mismatch
|
||||||
)
|
)
|
||||||
|
)
|
||||||
self._failure = {
|
self._failure = {
|
||||||
"code": type(exc).__name__,
|
"code": type(exc).__name__,
|
||||||
"reason_code": failure_reason_code,
|
"reason_code": failure_reason_code,
|
||||||
@@ -451,6 +548,37 @@ class InteractiveApplicationControlSession:
|
|||||||
else None
|
else None
|
||||||
),
|
),
|
||||||
"safe_to_retry": safe_to_retry,
|
"safe_to_retry": safe_to_retry,
|
||||||
|
**(
|
||||||
|
{
|
||||||
|
"status_reconciliation": {
|
||||||
|
"device_session_state": "ready",
|
||||||
|
"device_project_bound": False,
|
||||||
|
"system_error_code": None,
|
||||||
|
"decision": "safe-explicit-prestart-retry",
|
||||||
|
"automatic_retry": False,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if status_reconciled_prestart_failure
|
||||||
|
else {}
|
||||||
|
),
|
||||||
|
**(
|
||||||
|
{
|
||||||
|
"network_change_admissible": True,
|
||||||
|
"network_change_reconciliation": {
|
||||||
|
"device_session_state": "scan_stopping",
|
||||||
|
"device_project_bound": True,
|
||||||
|
"system_error_code": None,
|
||||||
|
"stop_complete": True,
|
||||||
|
"standby_confirmed": False,
|
||||||
|
"decision": (
|
||||||
|
"explicit-network-change-only-after-acknowledged-stop"
|
||||||
|
),
|
||||||
|
"automatic_retry": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if stop_acknowledged_network_change
|
||||||
|
else {}
|
||||||
|
),
|
||||||
}
|
}
|
||||||
self._outcome_unknown = outcome_unknown
|
self._outcome_unknown = outcome_unknown
|
||||||
self._dialogue_snapshot = dialogue_snapshot or None
|
self._dialogue_snapshot = dialogue_snapshot or None
|
||||||
@@ -462,7 +590,8 @@ class InteractiveApplicationControlSession:
|
|||||||
"publish_attempts=%s modeling_command_attempted=%s "
|
"publish_attempts=%s modeling_command_attempted=%s "
|
||||||
"diagnostic_snapshot_unavailable=%s "
|
"diagnostic_snapshot_unavailable=%s "
|
||||||
"diagnostic_evidence_unavailable=%s outcome_unknown=%s "
|
"diagnostic_evidence_unavailable=%s outcome_unknown=%s "
|
||||||
"safe_to_retry=%s",
|
"safe_to_retry=%s status_reconciliation=%s "
|
||||||
|
"network_change_reconciliation=%s",
|
||||||
type(exc).__name__,
|
type(exc).__name__,
|
||||||
failure_reason_code,
|
failure_reason_code,
|
||||||
failed_phase,
|
failed_phase,
|
||||||
@@ -474,6 +603,51 @@ class InteractiveApplicationControlSession:
|
|||||||
diagnostic_evidence_unavailable,
|
diagnostic_evidence_unavailable,
|
||||||
outcome_unknown,
|
outcome_unknown,
|
||||||
safe_to_retry,
|
safe_to_retry,
|
||||||
|
(
|
||||||
|
"safe-explicit-prestart-retry"
|
||||||
|
if status_reconciled_prestart_failure
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"explicit-network-change-only-after-acknowledged-stop"
|
||||||
|
if stop_acknowledged_network_change
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
extra={
|
||||||
|
"event_code": "k1_application_control_session_failed",
|
||||||
|
"reason_code": failure_reason_code,
|
||||||
|
"failed_phase": failed_phase,
|
||||||
|
"dialogue_stage": dialogue_stage,
|
||||||
|
"transport_state": self._json_string(
|
||||||
|
transport_snapshot.get("state")
|
||||||
|
),
|
||||||
|
"publish_attempts": publish_attempts,
|
||||||
|
"modeling_command_attempted": modeling_command_attempted,
|
||||||
|
"outcome_unknown": outcome_unknown,
|
||||||
|
"safe_to_retry": safe_to_retry,
|
||||||
|
"status_reconciliation": (
|
||||||
|
"safe-explicit-prestart-retry"
|
||||||
|
if status_reconciled_prestart_failure
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"network_change_admissible": (
|
||||||
|
stop_acknowledged_network_change
|
||||||
|
),
|
||||||
|
"network_change_reconciliation": (
|
||||||
|
"explicit-network-change-only-after-acknowledged-stop"
|
||||||
|
if stop_acknowledged_network_change
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"mqtt_loop_result_code": self._json_int_or_none(
|
||||||
|
transport_snapshot.get("last_loop_result_code")
|
||||||
|
),
|
||||||
|
"mqtt_loop_result_name": self._json_string(
|
||||||
|
transport_snapshot.get("last_loop_result_name")
|
||||||
|
),
|
||||||
|
"mqtt_loop_phase": self._json_string(
|
||||||
|
transport_snapshot.get("last_loop_phase")
|
||||||
|
),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
final_dialogue_snapshot: dict[str, object] | None = None
|
final_dialogue_snapshot: dict[str, object] | None = None
|
||||||
@@ -552,6 +726,23 @@ class InteractiveApplicationControlSession:
|
|||||||
with self._lock:
|
with self._lock:
|
||||||
self._set_phase_locked(phase)
|
self._set_phase_locked(phase)
|
||||||
|
|
||||||
|
def _notify_scanning_observer(self) -> None:
|
||||||
|
observer = self._scanning_observer
|
||||||
|
if observer is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
observer()
|
||||||
|
except Exception:
|
||||||
|
with self._lock:
|
||||||
|
self._scanning_observer_errors += 1
|
||||||
|
logger.exception(
|
||||||
|
"K1 confirmed SCANNING, but its acquisition observer failed",
|
||||||
|
extra={
|
||||||
|
"event_code": "k1_application_scanning_observer_failed",
|
||||||
|
"automatic_retry": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _executor_snapshot_safely(
|
def _executor_snapshot_safely(
|
||||||
executor: PhysicalAcceptanceDialogueExecutor | None,
|
executor: PhysicalAcceptanceDialogueExecutor | None,
|
||||||
|
|||||||
@@ -226,7 +226,7 @@ class VisualizationRuntime:
|
|||||||
raise RuntimeError("поток не завершился за отведённое время; повторите остановку")
|
raise RuntimeError("поток не завершился за отведённое время; повторите остановку")
|
||||||
|
|
||||||
def close(self, *, wait_seconds: float = 5.0) -> None:
|
def close(self, *, wait_seconds: float = 5.0) -> None:
|
||||||
"""Stop the active source and release the process-wide visual bridge."""
|
"""Stop the active source and release its acquisition-scoped bridge."""
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._closed = True
|
self._closed = True
|
||||||
thread = self._thread
|
thread = self._thread
|
||||||
@@ -429,13 +429,11 @@ class VisualizationRuntime:
|
|||||||
bridge: RerunBridge | None = None
|
bridge: RerunBridge | None = None
|
||||||
try:
|
try:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
bridge = self._bridge
|
|
||||||
if self._closed:
|
if self._closed:
|
||||||
publisher_aborted.set()
|
publisher_aborted.set()
|
||||||
if publisher_aborted.is_set():
|
if publisher_aborted.is_set():
|
||||||
publisher_ready.set()
|
publisher_ready.set()
|
||||||
return
|
return
|
||||||
if bridge is None:
|
|
||||||
candidate = self._bridge_factory(
|
candidate = self._bridge_factory(
|
||||||
grpc_port=self._grpc_port,
|
grpc_port=self._grpc_port,
|
||||||
metrics=self._metrics,
|
metrics=self._metrics,
|
||||||
@@ -446,6 +444,9 @@ class VisualizationRuntime:
|
|||||||
if self._closed:
|
if self._closed:
|
||||||
publisher_aborted.set()
|
publisher_aborted.set()
|
||||||
else:
|
else:
|
||||||
|
# Every acquisition owns a fresh Rerun recording/store.
|
||||||
|
# Reusing a RecordingStream across independent scans can
|
||||||
|
# leave a later WebViewer without a StoreInfo event.
|
||||||
self._bridge = candidate
|
self._bridge = candidate
|
||||||
if publisher_aborted.is_set():
|
if publisher_aborted.is_set():
|
||||||
publisher_ready.set()
|
publisher_ready.set()
|
||||||
@@ -520,19 +521,17 @@ class VisualizationRuntime:
|
|||||||
finally:
|
finally:
|
||||||
if bridge is not None:
|
if bridge is not None:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
close_bridge = self._closed and (
|
if self._bridge is bridge:
|
||||||
self._bridge is bridge or publisher_aborted.is_set()
|
|
||||||
)
|
|
||||||
if close_bridge and self._bridge is bridge:
|
|
||||||
self._bridge = None
|
self._bridge = None
|
||||||
self._rerun_grpc_url = None
|
self._rerun_grpc_url = None
|
||||||
if close_bridge:
|
|
||||||
try:
|
try:
|
||||||
bridge.close()
|
bridge.close()
|
||||||
except BaseException as exc:
|
except BaseException as exc:
|
||||||
# Process shutdown must not become a false successful
|
# A failed native disconnect must not become a false
|
||||||
# idle state when the native bridge failed to close.
|
# successful idle state or leak port 9876 into the next
|
||||||
|
# independent acquisition.
|
||||||
publisher_error.append(exc)
|
publisher_error.append(exc)
|
||||||
|
self._notify()
|
||||||
|
|
||||||
def join_publisher() -> None:
|
def join_publisher() -> None:
|
||||||
# Never orphan a publisher: the session thread remains its owner.
|
# Never orphan a publisher: the session thread remains its owner.
|
||||||
@@ -596,6 +595,7 @@ class VisualizationRuntime:
|
|||||||
self._message = message
|
self._message = message
|
||||||
self._foxglove_ws_url = None
|
self._foxglove_ws_url = None
|
||||||
self._foxglove_viewer_url = None
|
self._foxglove_viewer_url = None
|
||||||
|
self._rerun_grpc_url = None
|
||||||
self._notify()
|
self._notify()
|
||||||
|
|
||||||
def _finish_error(self, message: str) -> None:
|
def _finish_error(self, message: str) -> None:
|
||||||
@@ -605,6 +605,7 @@ class VisualizationRuntime:
|
|||||||
self._message = message
|
self._message = message
|
||||||
self._foxglove_ws_url = None
|
self._foxglove_ws_url = None
|
||||||
self._foxglove_viewer_url = None
|
self._foxglove_viewer_url = None
|
||||||
|
self._rerun_grpc_url = None
|
||||||
self._notify()
|
self._notify()
|
||||||
|
|
||||||
def _current_scene_settings(self) -> RerunSceneSettings:
|
def _current_scene_settings(self) -> RerunSceneSettings:
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
import math
|
import math
|
||||||
|
import socket
|
||||||
import time
|
import time
|
||||||
from collections.abc import Callable, Mapping
|
from collections.abc import Callable, Mapping
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
@@ -31,6 +33,7 @@ TRAJECTORY_MIN_DISTANCE_METERS = 0.02
|
|||||||
TRAJECTORY_PUBLISH_INTERVAL_NS = 500_000_000
|
TRAJECTORY_PUBLISH_INTERVAL_NS = 500_000_000
|
||||||
LIVE_GRPC_BUFFER_LIMIT = "32MiB"
|
LIVE_GRPC_BUFFER_LIMIT = "32MiB"
|
||||||
DEFAULT_GRPC_PORT = 9876
|
DEFAULT_GRPC_PORT = 9876
|
||||||
|
GRPC_PORT_SEARCH_SPAN = 128
|
||||||
DEFAULT_CORS_ORIGINS = (
|
DEFAULT_CORS_ORIGINS = (
|
||||||
"http://127.0.0.1:5173",
|
"http://127.0.0.1:5173",
|
||||||
"http://localhost:5173",
|
"http://localhost:5173",
|
||||||
@@ -40,6 +43,8 @@ DEFAULT_CORS_ORIGINS = (
|
|||||||
"http://localhost:8000",
|
"http://localhost:8000",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger("k1link.device_plugins.xgrids_k1.viewer_receiver")
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class RerunSceneSettings:
|
class RerunSceneSettings:
|
||||||
@@ -62,6 +67,34 @@ class RerunSceneSettings:
|
|||||||
SettingsProvider = Callable[[], RerunSceneSettings]
|
SettingsProvider = Callable[[], RerunSceneSettings]
|
||||||
|
|
||||||
|
|
||||||
|
def _select_available_grpc_port(
|
||||||
|
preferred_port: int,
|
||||||
|
*,
|
||||||
|
search_span: int = GRPC_PORT_SEARCH_SPAN,
|
||||||
|
) -> int:
|
||||||
|
"""Select a local Rerun port without reusing a still-served recording."""
|
||||||
|
|
||||||
|
if not 1 <= preferred_port <= 65_535:
|
||||||
|
raise ValueError("Rerun gRPC port must be between 1 and 65535")
|
||||||
|
if search_span < 1:
|
||||||
|
raise ValueError("Rerun gRPC port search span must be positive")
|
||||||
|
|
||||||
|
last_port = min(preferred_port + search_span, 65_536)
|
||||||
|
for candidate in range(preferred_port, last_port):
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
||||||
|
try:
|
||||||
|
# Rerun binds all local interfaces. Probe the same address class
|
||||||
|
# so a previous recording retained by a late viewer is detected.
|
||||||
|
probe.bind(("0.0.0.0", candidate))
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return candidate
|
||||||
|
raise RuntimeError(
|
||||||
|
"No local Rerun gRPC port is available in "
|
||||||
|
f"{preferred_port}..{last_port - 1}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class RerunBridge:
|
class RerunBridge:
|
||||||
"""Publish transport-neutral canonical envelopes to a Rerun recording."""
|
"""Publish transport-neutral canonical envelopes to a Rerun recording."""
|
||||||
|
|
||||||
@@ -85,9 +118,20 @@ class RerunBridge:
|
|||||||
else:
|
else:
|
||||||
recording = recording_factory("nodedc_mission_core_spatial")
|
recording = recording_factory("nodedc_mission_core_spatial")
|
||||||
try:
|
try:
|
||||||
|
selected_grpc_port = _select_available_grpc_port(grpc_port)
|
||||||
|
if selected_grpc_port != grpc_port:
|
||||||
|
logger.info(
|
||||||
|
"Mission Core selected a new Rerun port because an earlier "
|
||||||
|
"viewer still owns the preferred listener",
|
||||||
|
extra={
|
||||||
|
"event_code": "rerun_grpc_port_rotated",
|
||||||
|
"preferred_port": grpc_port,
|
||||||
|
"selected_port": selected_grpc_port,
|
||||||
|
},
|
||||||
|
)
|
||||||
blueprint = _blueprint(self._settings)
|
blueprint = _blueprint(self._settings)
|
||||||
url = recording.serve_grpc(
|
url = recording.serve_grpc(
|
||||||
grpc_port=grpc_port,
|
grpc_port=selected_grpc_port,
|
||||||
default_blueprint=blueprint,
|
default_blueprint=blueprint,
|
||||||
# This is a reconnect cushion for the live preview, not the source
|
# This is a reconnect cushion for the live preview, not the source
|
||||||
# of record. Raw MQTT evidence is persisted independently. A large
|
# of record. Raw MQTT evidence is persisted independently. A large
|
||||||
@@ -132,7 +176,7 @@ class RerunBridge:
|
|||||||
return self._url
|
return self._url
|
||||||
|
|
||||||
def begin_session(self, metrics: BridgeMetrics | None = None) -> None:
|
def begin_session(self, metrics: BridgeMetrics | None = None) -> None:
|
||||||
"""Reset session state while keeping the process-lifetime server alive."""
|
"""Initialize the one acquisition owned by this recording server."""
|
||||||
if self._closed:
|
if self._closed:
|
||||||
raise RuntimeError("Rerun bridge is already closed")
|
raise RuntimeError("Rerun bridge is already closed")
|
||||||
if metrics is not None:
|
if metrics is not None:
|
||||||
|
|||||||
@@ -53,8 +53,10 @@ from k1link.web.plugin_runtime import (
|
|||||||
PluginRuntimeUnavailableError,
|
PluginRuntimeUnavailableError,
|
||||||
)
|
)
|
||||||
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
|
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
|
||||||
|
from k1link.web.runtime_diagnostics import configure_scanner_diagnostics
|
||||||
from k1link.web.session_api import build_session_router
|
from k1link.web.session_api import build_session_router
|
||||||
from k1link.web.system_telemetry_api import build_system_telemetry_router
|
from k1link.web.system_telemetry_api import build_system_telemetry_router
|
||||||
|
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
|
||||||
|
|
||||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
||||||
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
|
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
|
||||||
@@ -274,6 +276,9 @@ async def _recording_preparation_reconciler() -> None:
|
|||||||
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||||
reconciler: asyncio.Task[None] | None = None
|
reconciler: asyncio.Task[None] | None = None
|
||||||
try:
|
try:
|
||||||
|
configure_scanner_diagnostics(
|
||||||
|
REPOSITORY_ROOT / ".runtime" / "mission-core" / "logs"
|
||||||
|
)
|
||||||
session_recording_preparation_manager.start()
|
session_recording_preparation_manager.start()
|
||||||
# Recovery is intentionally a one-shot startup phase. The archive
|
# Recovery is intentionally a one-shot startup phase. The archive
|
||||||
# helper owns a cross-process lease, while ordinary catalog requests
|
# helper owns a cross-process lease, while ordinary catalog requests
|
||||||
@@ -671,6 +676,7 @@ app.include_router(
|
|||||||
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
app.include_router(build_viewer_diagnostics_router())
|
||||||
|
|
||||||
|
|
||||||
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from logging.handlers import RotatingFileHandler
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
|
SCANNER_LOGGER_NAME: Final = "k1link.device_plugins.xgrids_k1"
|
||||||
|
SCANNER_DIAGNOSTIC_FILE: Final = "scanner-diagnostics.jsonl"
|
||||||
|
_HANDLER_MARKER: Final = "_mission_core_scanner_diagnostics_path"
|
||||||
|
_EXTRA_FIELDS: Final = (
|
||||||
|
"event_code",
|
||||||
|
"reason_code",
|
||||||
|
"failed_phase",
|
||||||
|
"dialogue_stage",
|
||||||
|
"transport_state",
|
||||||
|
"publish_attempts",
|
||||||
|
"modeling_command_attempted",
|
||||||
|
"outcome_unknown",
|
||||||
|
"safe_to_retry",
|
||||||
|
"status_reconciliation",
|
||||||
|
"network_change_admissible",
|
||||||
|
"network_change_reconciliation",
|
||||||
|
"mqtt_loop_result_code",
|
||||||
|
"mqtt_loop_result_name",
|
||||||
|
"mqtt_loop_phase",
|
||||||
|
"automatic_retry",
|
||||||
|
"camera_source_id",
|
||||||
|
"evidence_session_id",
|
||||||
|
"activation_trigger",
|
||||||
|
"lease_generation",
|
||||||
|
"lease_state",
|
||||||
|
"recovery_strategy",
|
||||||
|
"endpoint_reachable",
|
||||||
|
"host_route_class",
|
||||||
|
"address_changed",
|
||||||
|
"device_write_performed",
|
||||||
|
"failure_stage",
|
||||||
|
"start_checkpoint_released",
|
||||||
|
"stale_session_retired",
|
||||||
|
"stream_id",
|
||||||
|
"backend_activity_sequence",
|
||||||
|
"viewer_range_max_ns",
|
||||||
|
"stalled_for_ms",
|
||||||
|
"recovery_attempt",
|
||||||
|
"preferred_port",
|
||||||
|
"selected_port",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ScannerDiagnosticJsonFormatter(logging.Formatter):
|
||||||
|
"""Serialize a bounded, secret-free scanner diagnostic record."""
|
||||||
|
|
||||||
|
def format(self, record: logging.LogRecord) -> str:
|
||||||
|
document: dict[str, object] = {
|
||||||
|
"timestamp_utc": datetime.fromtimestamp(record.created, UTC)
|
||||||
|
.isoformat()
|
||||||
|
.replace("+00:00", "Z"),
|
||||||
|
"level": record.levelname,
|
||||||
|
"logger": record.name,
|
||||||
|
"message": record.getMessage()[:1000],
|
||||||
|
}
|
||||||
|
for field in _EXTRA_FIELDS:
|
||||||
|
value = getattr(record, field, None)
|
||||||
|
if value is not None and isinstance(value, (str, int, bool)):
|
||||||
|
document[field] = value
|
||||||
|
return json.dumps(document, ensure_ascii=False, separators=(",", ":"))
|
||||||
|
|
||||||
|
|
||||||
|
def configure_scanner_diagnostics(logs_root: Path) -> Path:
|
||||||
|
"""Install one private rotating JSONL handler for K1 field diagnostics."""
|
||||||
|
|
||||||
|
logs_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
os.chmod(logs_root, 0o700)
|
||||||
|
target = (logs_root / SCANNER_DIAGNOSTIC_FILE).resolve()
|
||||||
|
logger = logging.getLogger(SCANNER_LOGGER_NAME)
|
||||||
|
for handler in logger.handlers:
|
||||||
|
if getattr(handler, _HANDLER_MARKER, None) == str(target):
|
||||||
|
return target
|
||||||
|
|
||||||
|
handler = RotatingFileHandler(
|
||||||
|
target,
|
||||||
|
maxBytes=5 * 1024 * 1024,
|
||||||
|
backupCount=5,
|
||||||
|
encoding="utf-8",
|
||||||
|
delay=False,
|
||||||
|
)
|
||||||
|
os.chmod(target, 0o600)
|
||||||
|
handler.setLevel(logging.INFO)
|
||||||
|
handler.setFormatter(ScannerDiagnosticJsonFormatter())
|
||||||
|
setattr(handler, _HANDLER_MARKER, str(target))
|
||||||
|
logger.addHandler(handler)
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
return target
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Response
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
|
logger = logging.getLogger("k1link.device_plugins.xgrids_k1.viewer_receiver")
|
||||||
|
|
||||||
|
LiveViewerEventCode = Literal[
|
||||||
|
"live_receiver_stalled",
|
||||||
|
"live_receiver_restart_requested",
|
||||||
|
"live_receiver_recovered",
|
||||||
|
"live_receiver_recovery_exhausted",
|
||||||
|
"live_receiver_active_store_admitted",
|
||||||
|
"live_receiver_error",
|
||||||
|
]
|
||||||
|
LiveViewerFailureStage = Literal[
|
||||||
|
"recording-open-timeout",
|
||||||
|
"viewer-start",
|
||||||
|
"module-load",
|
||||||
|
"receiver-stalled",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class LiveViewerDiagnosticEvent(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid", strict=True)
|
||||||
|
|
||||||
|
schema_version: Literal["missioncore.live-viewer-diagnostic/v1"]
|
||||||
|
event_code: LiveViewerEventCode
|
||||||
|
failure_stage: LiveViewerFailureStage | None = None
|
||||||
|
stream_id: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
min_length=1,
|
||||||
|
max_length=128,
|
||||||
|
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$",
|
||||||
|
)
|
||||||
|
backend_activity_sequence: int | None = Field(default=None, ge=0)
|
||||||
|
viewer_range_max_ns: int | None = Field(default=None, ge=0)
|
||||||
|
stalled_for_ms: int | None = Field(default=None, ge=0, le=600_000)
|
||||||
|
recovery_attempt: int | None = Field(default=None, ge=1, le=3)
|
||||||
|
|
||||||
|
|
||||||
|
def build_viewer_diagnostics_router() -> APIRouter:
|
||||||
|
router = APIRouter(prefix="/api/v1/viewer", tags=["viewer"])
|
||||||
|
|
||||||
|
@router.post("/live-diagnostics", status_code=204)
|
||||||
|
def record_live_diagnostic(event: LiveViewerDiagnosticEvent) -> Response:
|
||||||
|
logger.info(
|
||||||
|
"Mission Core live Rerun receiver diagnostic: event=%s stream=%s",
|
||||||
|
event.event_code,
|
||||||
|
event.stream_id,
|
||||||
|
extra={
|
||||||
|
"event_code": event.event_code,
|
||||||
|
"failure_stage": event.failure_stage,
|
||||||
|
"stream_id": event.stream_id,
|
||||||
|
"backend_activity_sequence": event.backend_activity_sequence,
|
||||||
|
"viewer_range_max_ns": event.viewer_range_max_ns,
|
||||||
|
"stalled_for_ms": event.stalled_for_ms,
|
||||||
|
"recovery_attempt": event.recovery_attempt,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return Response(status_code=204)
|
||||||
|
|
||||||
|
return router
|
||||||
@@ -136,6 +136,31 @@ def test_live_ingress_allows_only_one_worker_consumer() -> None:
|
|||||||
ingress.open_consumer("worker-2")
|
ingress.open_consumer("worker-2")
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_ingress_new_session_discards_queued_events_from_previous_session() -> None:
|
||||||
|
ingress = LivePerceptionIngress()
|
||||||
|
ingress.open_consumer("worker-1")
|
||||||
|
ingress.begin_session("session-1")
|
||||||
|
assert ingress.publish(
|
||||||
|
modality="lidar",
|
||||||
|
source_id="lixel/application/report/lio_pcl",
|
||||||
|
source_sequence=1,
|
||||||
|
captured_at_epoch_ns=1,
|
||||||
|
received_monotonic_ns=1,
|
||||||
|
payload=b"old-session",
|
||||||
|
)
|
||||||
|
|
||||||
|
ingress.end_session("session-1")
|
||||||
|
ingress.begin_session("session-2")
|
||||||
|
|
||||||
|
events = []
|
||||||
|
while (event := ingress.take_next("worker-1", timeout=0)) is not None:
|
||||||
|
events.append(event)
|
||||||
|
assert len(events) == 1
|
||||||
|
assert events[0].session_id == "session-2"
|
||||||
|
assert events[0].modality == "control"
|
||||||
|
assert events[0].payload == b'{"event":"session-start"}'
|
||||||
|
|
||||||
|
|
||||||
def test_live_result_round_trip_keeps_video_mask_boxes_and_shadow_authority() -> None:
|
def test_live_result_round_trip_keeps_video_mask_boxes_and_shadow_authority() -> None:
|
||||||
mask = np.zeros((600, 800), dtype=np.uint8)
|
mask = np.zeros((600, 800), dtype=np.uint8)
|
||||||
mask[100:120, 200:240] = 4
|
mask[100:120, 200:240] = 4
|
||||||
|
|||||||
+40
-16
@@ -1,6 +1,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
|
import socket
|
||||||
import struct
|
import struct
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
@@ -20,6 +22,7 @@ from k1link.viewer.rerun_bridge import (
|
|||||||
RerunSceneSettings,
|
RerunSceneSettings,
|
||||||
_live_time_panel,
|
_live_time_panel,
|
||||||
_point_colors,
|
_point_colors,
|
||||||
|
_select_available_grpc_port,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -62,6 +65,33 @@ class DisconnectFailureRecording(FakeRecording):
|
|||||||
raise RuntimeError("synthetic disconnect failure")
|
raise RuntimeError("synthetic disconnect failure")
|
||||||
|
|
||||||
|
|
||||||
|
def test_rerun_port_selection_skips_a_recording_still_held_by_a_viewer(
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as occupied:
|
||||||
|
occupied.bind(("0.0.0.0", 0))
|
||||||
|
occupied.listen()
|
||||||
|
preferred_port = int(occupied.getsockname()[1])
|
||||||
|
|
||||||
|
selected_port = _select_available_grpc_port(preferred_port, search_span=8)
|
||||||
|
recording = FakeRecording()
|
||||||
|
with caplog.at_level(
|
||||||
|
logging.INFO,
|
||||||
|
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
||||||
|
):
|
||||||
|
bridge = RerunBridge(
|
||||||
|
grpc_port=preferred_port,
|
||||||
|
recording_factory=lambda _: recording, # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert selected_port != preferred_port
|
||||||
|
assert preferred_port < selected_port < preferred_port + 8
|
||||||
|
assert caplog.records[-1].event_code == "rerun_grpc_port_rotated"
|
||||||
|
assert caplog.records[-1].preferred_port == preferred_port
|
||||||
|
assert caplog.records[-1].selected_port == selected_port
|
||||||
|
bridge.close()
|
||||||
|
|
||||||
|
|
||||||
def _message(
|
def _message(
|
||||||
topic: str,
|
topic: str,
|
||||||
payload: bytes,
|
payload: bytes,
|
||||||
@@ -266,7 +296,7 @@ def test_palettes_are_deterministic_and_custom_color_is_exact() -> None:
|
|||||||
assert custom_over_rgb.tolist() == [[16, 32, 48], [16, 32, 48]]
|
assert custom_over_rgb.tolist() == [[16, 32, 48], [16, 32, 48]]
|
||||||
|
|
||||||
|
|
||||||
def test_runtime_reuses_one_bridge_across_sequential_sessions(tmp_path: Path) -> None:
|
def test_runtime_owns_fresh_bridge_for_each_sequential_session(tmp_path: Path) -> None:
|
||||||
capture = tmp_path / "mqtt.raw.k1mqtt"
|
capture = tmp_path / "mqtt.raw.k1mqtt"
|
||||||
point_topic = "RealtimePointcloud"
|
point_topic = "RealtimePointcloud"
|
||||||
pose_topic = "RealtimePath"
|
pose_topic = "RealtimePath"
|
||||||
@@ -300,10 +330,12 @@ def test_runtime_reuses_one_bridge_across_sequential_sessions(tmp_path: Path) ->
|
|||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
|
|
||||||
recording = FakeRecording()
|
recordings: list[FakeRecording] = []
|
||||||
created: list[RerunBridge] = []
|
created: list[RerunBridge] = []
|
||||||
|
|
||||||
def bridge_factory(**kwargs: object) -> RerunBridge:
|
def bridge_factory(**kwargs: object) -> RerunBridge:
|
||||||
|
recording = FakeRecording()
|
||||||
|
recordings.append(recording)
|
||||||
bridge = RerunBridge(
|
bridge = RerunBridge(
|
||||||
recording_factory=lambda _: recording, # type: ignore[arg-type]
|
recording_factory=lambda _: recording, # type: ignore[arg-type]
|
||||||
**kwargs, # type: ignore[arg-type]
|
**kwargs, # type: ignore[arg-type]
|
||||||
@@ -329,8 +361,8 @@ def test_runtime_reuses_one_bridge_across_sequential_sessions(tmp_path: Path) ->
|
|||||||
assert snapshot["metrics"]["mqtt_to_publish_ms"] is None
|
assert snapshot["metrics"]["mqtt_to_publish_ms"] is None
|
||||||
runtime.stop(wait_seconds=5.0)
|
runtime.stop(wait_seconds=5.0)
|
||||||
assert runtime.snapshot()["phase"] == "idle"
|
assert runtime.snapshot()["phase"] == "idle"
|
||||||
assert runtime.snapshot()["rerun_grpc_url"] == "rerun+http://127.0.0.1:9876/proxy"
|
assert runtime.snapshot()["rerun_grpc_url"] is None
|
||||||
assert recording.disconnected is False
|
assert recordings[0].disconnected is True
|
||||||
|
|
||||||
runtime.start_replay(capture, speed=0.0)
|
runtime.start_replay(capture, speed=0.0)
|
||||||
deadline = time.monotonic() + 5.0
|
deadline = time.monotonic() + 5.0
|
||||||
@@ -340,13 +372,12 @@ def test_runtime_reuses_one_bridge_across_sequential_sessions(tmp_path: Path) ->
|
|||||||
time.sleep(0.05)
|
time.sleep(0.05)
|
||||||
runtime.stop(wait_seconds=5.0)
|
runtime.stop(wait_seconds=5.0)
|
||||||
|
|
||||||
assert len(created) == 1
|
assert len(created) == 2
|
||||||
assert runtime.snapshot()["metrics"]["pcl_frames"] == 1
|
assert runtime.snapshot()["metrics"]["pcl_frames"] == 1
|
||||||
assert runtime.snapshot()["rerun_grpc_url"] == "rerun+http://127.0.0.1:9876/proxy"
|
assert runtime.snapshot()["rerun_grpc_url"] is None
|
||||||
assert recording.disconnected is False
|
assert all(recording.disconnected for recording in recordings)
|
||||||
runtime.close()
|
runtime.close()
|
||||||
assert runtime.snapshot()["rerun_grpc_url"] is None
|
assert runtime.snapshot()["rerun_grpc_url"] is None
|
||||||
assert recording.disconnected is True
|
|
||||||
|
|
||||||
|
|
||||||
def test_runtime_reports_bridge_close_failure_instead_of_false_idle(tmp_path: Path) -> None:
|
def test_runtime_reports_bridge_close_failure_instead_of_false_idle(tmp_path: Path) -> None:
|
||||||
@@ -385,18 +416,11 @@ def test_runtime_reports_bridge_close_failure_instead_of_false_idle(tmp_path: Pa
|
|||||||
time.sleep(0.01)
|
time.sleep(0.01)
|
||||||
snapshot = runtime.snapshot()
|
snapshot = runtime.snapshot()
|
||||||
|
|
||||||
assert snapshot["phase"] == "idle"
|
|
||||||
assert snapshot["rerun_grpc_url"] == "rerun+http://127.0.0.1:9876/proxy"
|
|
||||||
assert recording.disconnected is False
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="synthetic disconnect failure"):
|
|
||||||
runtime.close()
|
|
||||||
|
|
||||||
snapshot = runtime.snapshot()
|
|
||||||
assert snapshot["phase"] == "error"
|
assert snapshot["phase"] == "error"
|
||||||
assert "synthetic disconnect failure" in snapshot["message"]
|
assert "synthetic disconnect failure" in snapshot["message"]
|
||||||
assert snapshot["rerun_grpc_url"] is None
|
assert snapshot["rerun_grpc_url"] is None
|
||||||
assert recording.disconnected is True
|
assert recording.disconnected is True
|
||||||
|
runtime.close()
|
||||||
|
|
||||||
|
|
||||||
def test_close_during_blocked_factory_closes_the_late_bridge(tmp_path: Path) -> None:
|
def test_close_during_blocked_factory_closes_the_late_bridge(tmp_path: Path) -> None:
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import stat
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import APIRouter
|
||||||
|
from fastapi.routing import APIRoute
|
||||||
|
from pydantic import ValidationError
|
||||||
|
|
||||||
|
from k1link.web.runtime_diagnostics import (
|
||||||
|
SCANNER_LOGGER_NAME,
|
||||||
|
configure_scanner_diagnostics,
|
||||||
|
)
|
||||||
|
from k1link.web.viewer_diagnostics_api import (
|
||||||
|
LiveViewerDiagnosticEvent,
|
||||||
|
build_viewer_diagnostics_router,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _endpoint(router: APIRouter, path: str, method: str) -> Callable[..., Any]:
|
||||||
|
for route in router.routes:
|
||||||
|
if (
|
||||||
|
isinstance(route, APIRoute)
|
||||||
|
and route.path == path
|
||||||
|
and route.methods is not None
|
||||||
|
and method in route.methods
|
||||||
|
):
|
||||||
|
return route.endpoint
|
||||||
|
raise AssertionError(f"{method} {path} route is missing")
|
||||||
|
|
||||||
|
|
||||||
|
def test_private_scanner_diagnostics_are_durable_structured_and_bounded(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
target = configure_scanner_diagnostics(tmp_path / "logs")
|
||||||
|
logger = logging.getLogger(f"{SCANNER_LOGGER_NAME}.test")
|
||||||
|
logger.error(
|
||||||
|
"field control failure",
|
||||||
|
extra={
|
||||||
|
"event_code": "k1_application_control_session_failed",
|
||||||
|
"reason_code": "mqtt_network_loop_failed",
|
||||||
|
"mqtt_loop_result_code": 7,
|
||||||
|
"mqtt_loop_result_name": "The connection was lost.",
|
||||||
|
"mqtt_loop_phase": "post-publish-drain",
|
||||||
|
"automatic_retry": False,
|
||||||
|
"camera_source_id": "sensor.camera.right",
|
||||||
|
"evidence_session_id": "20260728T163450Z_viewer_live",
|
||||||
|
"activation_trigger": "application-control-scanning",
|
||||||
|
"network_change_admissible": True,
|
||||||
|
"network_change_reconciliation": (
|
||||||
|
"explicit-network-change-only-after-acknowledged-stop"
|
||||||
|
),
|
||||||
|
"lease_generation": 3,
|
||||||
|
"lease_state": "reachable",
|
||||||
|
"recovery_strategy": "existing-mqtt-endpoint",
|
||||||
|
"endpoint_reachable": True,
|
||||||
|
"address_changed": False,
|
||||||
|
"device_write_performed": False,
|
||||||
|
"preferred_port": 9876,
|
||||||
|
"selected_port": 9877,
|
||||||
|
"unapproved_secret_field": "must-not-be-written",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
for handler in logging.getLogger(SCANNER_LOGGER_NAME).handlers:
|
||||||
|
handler.flush()
|
||||||
|
|
||||||
|
document = json.loads(target.read_text(encoding="utf-8").splitlines()[-1])
|
||||||
|
assert stat.S_IMODE(target.parent.stat().st_mode) == 0o700
|
||||||
|
assert stat.S_IMODE(target.stat().st_mode) == 0o600
|
||||||
|
assert document["event_code"] == "k1_application_control_session_failed"
|
||||||
|
assert document["reason_code"] == "mqtt_network_loop_failed"
|
||||||
|
assert document["mqtt_loop_result_code"] == 7
|
||||||
|
assert document["mqtt_loop_phase"] == "post-publish-drain"
|
||||||
|
assert document["automatic_retry"] is False
|
||||||
|
assert document["camera_source_id"] == "sensor.camera.right"
|
||||||
|
assert document["evidence_session_id"] == "20260728T163450Z_viewer_live"
|
||||||
|
assert document["activation_trigger"] == "application-control-scanning"
|
||||||
|
assert document["network_change_admissible"] is True
|
||||||
|
assert document["network_change_reconciliation"] == (
|
||||||
|
"explicit-network-change-only-after-acknowledged-stop"
|
||||||
|
)
|
||||||
|
assert document["lease_generation"] == 3
|
||||||
|
assert document["lease_state"] == "reachable"
|
||||||
|
assert document["recovery_strategy"] == "existing-mqtt-endpoint"
|
||||||
|
assert document["endpoint_reachable"] is True
|
||||||
|
assert document["address_changed"] is False
|
||||||
|
assert document["device_write_performed"] is False
|
||||||
|
assert document["preferred_port"] == 9876
|
||||||
|
assert document["selected_port"] == 9877
|
||||||
|
assert "unapproved_secret_field" not in document
|
||||||
|
|
||||||
|
parent = logging.getLogger(SCANNER_LOGGER_NAME)
|
||||||
|
for handler in list(parent.handlers):
|
||||||
|
if getattr(handler, "baseFilename", None) == str(target):
|
||||||
|
parent.removeHandler(handler)
|
||||||
|
handler.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_viewer_diagnostic_endpoint_accepts_only_bounded_events(
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
router = build_viewer_diagnostics_router()
|
||||||
|
endpoint = _endpoint(router, "/api/v1/viewer/live-diagnostics", "POST")
|
||||||
|
event = LiveViewerDiagnosticEvent(
|
||||||
|
schema_version="missioncore.live-viewer-diagnostic/v1",
|
||||||
|
event_code="live_receiver_stalled",
|
||||||
|
failure_stage="receiver-stalled",
|
||||||
|
stream_id="acquisition-123",
|
||||||
|
backend_activity_sequence=8_572,
|
||||||
|
viewer_range_max_ns=231_000_000_000,
|
||||||
|
stalled_for_ms=5_500,
|
||||||
|
recovery_attempt=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
with caplog.at_level(
|
||||||
|
logging.INFO,
|
||||||
|
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
||||||
|
):
|
||||||
|
response = endpoint(event)
|
||||||
|
|
||||||
|
assert response.status_code == 204
|
||||||
|
assert "event=live_receiver_stalled" in caplog.text
|
||||||
|
assert caplog.records[-1].failure_stage == "receiver-stalled"
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
LiveViewerDiagnosticEvent.model_validate(
|
||||||
|
{
|
||||||
|
**event.model_dump(),
|
||||||
|
"source_url": "http://192.168.56.1/private",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
fallback = LiveViewerDiagnosticEvent(
|
||||||
|
schema_version="missioncore.live-viewer-diagnostic/v1",
|
||||||
|
event_code="live_receiver_active_store_admitted",
|
||||||
|
stream_id="acquisition-123",
|
||||||
|
backend_activity_sequence=8_573,
|
||||||
|
)
|
||||||
|
assert fallback.failure_stage is None
|
||||||
@@ -128,3 +128,60 @@ def test_read_wifi_status_once_reads_only_and_returns_current_dhcp_address(
|
|||||||
assert result["operation"] == "single_reviewed_wifi_status_read"
|
assert result["operation"] == "single_reviewed_wifi_status_read"
|
||||||
assert result["write_performed"] is False
|
assert result["write_performed"] is False
|
||||||
assert result["status"]["ipv4"] == "10.255.254.77"
|
assert result["status"]["ipv4"] == "10.255.254.77"
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_wifi_status_recovery_rediscover_ignores_retained_handle(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
value = bytearray(54)
|
||||||
|
value[0] = 11
|
||||||
|
value[1:12] = b"WIFI_CLIENT"
|
||||||
|
value[33] = 4
|
||||||
|
value[34:38] = bytes((10, 255, 254, 77))
|
||||||
|
value[50] = 1
|
||||||
|
stale_handle = object()
|
||||||
|
recovered_handle = object()
|
||||||
|
characteristic = SimpleNamespace(
|
||||||
|
uuid=wifi_module.STATUS_CHARACTERISTIC_UUID,
|
||||||
|
service_uuid=wifi_module.SERVICE_UUID,
|
||||||
|
properties=["read"],
|
||||||
|
)
|
||||||
|
service = SimpleNamespace(uuid=wifi_module.SERVICE_UUID)
|
||||||
|
|
||||||
|
class FakeServices:
|
||||||
|
def get_service(self, uuid: str) -> object | None:
|
||||||
|
return service if uuid == wifi_module.SERVICE_UUID else None
|
||||||
|
|
||||||
|
def get_characteristic(self, uuid: str) -> object | None:
|
||||||
|
return characteristic if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID else None
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
def __init__(self, device: object, **_kwargs: object) -> None:
|
||||||
|
assert device is recovered_handle
|
||||||
|
self.services = FakeServices()
|
||||||
|
self.name = "XGR-K1"
|
||||||
|
|
||||||
|
async def __aenter__(self) -> Any:
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *_args: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def read_gatt_char(self, _characteristic: object) -> bytes:
|
||||||
|
return bytes(value)
|
||||||
|
|
||||||
|
async def rediscover(*_args: object, **_kwargs: object) -> object:
|
||||||
|
return recovered_handle
|
||||||
|
|
||||||
|
monkeypatch.setattr(wifi_module, "discovered_device", lambda _uuid: stale_handle)
|
||||||
|
monkeypatch.setattr(wifi_module.BleakScanner, "find_device_by_address", rediscover)
|
||||||
|
monkeypatch.setattr(wifi_module, "BleakClient", FakeClient)
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
read_wifi_status_once(
|
||||||
|
"synthetic-corebluetooth-uuid",
|
||||||
|
rediscover=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"]["ipv4"] == "10.255.254.77"
|
||||||
|
|||||||
@@ -247,6 +247,8 @@ def test_verify_connection_refreshes_dynamic_dhcp_address_and_rotates_session(
|
|||||||
assert state["device_session"]["device_session_id"] != "old-device-session"
|
assert state["device_session"]["device_session_id"] != "old-device-session"
|
||||||
assert state["connection_verification"] == {
|
assert state["connection_verification"] == {
|
||||||
"status": "live-address-observed",
|
"status": "live-address-observed",
|
||||||
|
"lease_state": "configured",
|
||||||
|
"lease_generation": 1,
|
||||||
"endpoint_validation": "ble-wifi-status-read",
|
"endpoint_validation": "ble-wifi-status-read",
|
||||||
"network_reachability": "not-probed",
|
"network_reachability": "not-probed",
|
||||||
"address_changed": True,
|
"address_changed": True,
|
||||||
@@ -283,7 +285,7 @@ def test_implicit_acquisition_target_uses_current_ble_dhcp_address(
|
|||||||
assert state["acquisition"]["target_host"] == "10.255.254.77"
|
assert state["acquisition"]["target_host"] == "10.255.254.77"
|
||||||
|
|
||||||
|
|
||||||
def test_control_session_opens_against_current_ble_dhcp_address(
|
def test_control_session_reuses_reachable_process_owned_connection_lease(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -291,6 +293,8 @@ def test_control_session_opens_against_current_ble_dhcp_address(
|
|||||||
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
||||||
service._connection_mode = "bridge" # noqa: SLF001
|
service._connection_mode = "bridge" # noqa: SLF001
|
||||||
service._k1_ip = "10.255.254.54" # noqa: SLF001
|
service._k1_ip = "10.255.254.54" # noqa: SLF001
|
||||||
|
service._device_id = "known-k1" # noqa: SLF001
|
||||||
|
service._device_session_id = "known-session" # noqa: SLF001
|
||||||
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||||
opened_hosts: list[str] = []
|
opened_hosts: list[str] = []
|
||||||
|
|
||||||
@@ -302,12 +306,12 @@ def test_control_session_opens_against_current_ble_dhcp_address(
|
|||||||
opened_hosts.append(host)
|
opened_hosts.append(host)
|
||||||
return self.snapshot()
|
return self.snapshot()
|
||||||
|
|
||||||
async def fake_status_read(*_: object, **__: object) -> dict[str, Any]:
|
async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]:
|
||||||
return _wifi_status_read("10.255.254.77")
|
raise AssertionError("reachable connection lease must not reopen BLE")
|
||||||
|
|
||||||
service._application_control_session = FakeOpenControlSession() # type: ignore[assignment] # noqa: SLF001
|
service._application_control_session = FakeOpenControlSession() # type: ignore[assignment] # noqa: SLF001
|
||||||
monkeypatch.setattr(facade_module, "read_wifi_status_once", fake_status_read)
|
monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read)
|
||||||
monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False)
|
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True)
|
||||||
|
|
||||||
state = service.open_application_control_session(
|
state = service.open_application_control_session(
|
||||||
OpenApplicationControlSessionRequest(
|
OpenApplicationControlSessionRequest(
|
||||||
@@ -320,8 +324,241 @@ def test_control_session_opens_against_current_ble_dhcp_address(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
assert opened_hosts == ["10.255.254.54"]
|
||||||
|
assert state["k1_ip"] == "10.255.254.54"
|
||||||
|
assert state["connection_verification"]["lease_state"] == "reachable"
|
||||||
|
assert state["connection_verification"]["network_reachability"] == "reachable"
|
||||||
|
open_operation = next(
|
||||||
|
operation
|
||||||
|
for operation in state["operations"]
|
||||||
|
if operation["action"] == "application-control.session.open"
|
||||||
|
)
|
||||||
|
assert open_operation["status"] == "succeeded"
|
||||||
|
assert open_operation["result"] == {
|
||||||
|
"lease_generation": 0,
|
||||||
|
"connection_lease_reused": True,
|
||||||
|
"recovery_performed": False,
|
||||||
|
"address_changed": False,
|
||||||
|
"device_write_performed": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_reachable_connection_lease_supports_repeated_independent_control_sessions(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, _ = service_with_fake_runtime(tmp_path)
|
||||||
|
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
||||||
|
service._connection_mode = "bridge" # noqa: SLF001
|
||||||
|
service._k1_ip = "10.255.254.54" # noqa: SLF001
|
||||||
|
service._device_id = "known-k1" # noqa: SLF001
|
||||||
|
service._device_session_id = "known-session" # noqa: SLF001
|
||||||
|
service._connection_lease_generation = 7 # noqa: SLF001
|
||||||
|
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||||
|
opened_hosts: list[str] = []
|
||||||
|
|
||||||
|
class FakeCompletedControlSession:
|
||||||
|
def snapshot(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"state": "completed",
|
||||||
|
"can_open": True,
|
||||||
|
"can_confirm_standby": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
def open(self, *, host: str, **_: object) -> dict[str, object]:
|
||||||
|
opened_hosts.append(host)
|
||||||
|
return self.snapshot()
|
||||||
|
|
||||||
|
async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]:
|
||||||
|
raise AssertionError("repeated scans must not repeat BLE or Wi-Fi setup")
|
||||||
|
|
||||||
|
service._application_control_session = FakeCompletedControlSession() # type: ignore[assignment] # noqa: SLF001
|
||||||
|
monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read)
|
||||||
|
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True)
|
||||||
|
request = OpenApplicationControlSessionRequest(
|
||||||
|
operator_present=True,
|
||||||
|
owner_controlled_device=True,
|
||||||
|
lixelgo_closed=True,
|
||||||
|
battery_storage_confirmed=True,
|
||||||
|
expected_physical_state_confirmed=True,
|
||||||
|
timezone_name="Europe/Moscow",
|
||||||
|
)
|
||||||
|
|
||||||
|
service.open_application_control_session(request)
|
||||||
|
state = service.open_application_control_session(request)
|
||||||
|
|
||||||
|
assert opened_hosts == ["10.255.254.54", "10.255.254.54"]
|
||||||
|
assert state["device_session"]["device_session_id"] == "known-session"
|
||||||
|
assert state["connection_verification"]["lease_generation"] == 7
|
||||||
|
open_operations = [
|
||||||
|
operation
|
||||||
|
for operation in state["operations"]
|
||||||
|
if operation["action"] == "application-control.session.open"
|
||||||
|
]
|
||||||
|
assert len(open_operations) == 2
|
||||||
|
assert {operation["status"] for operation in open_operations} == {"succeeded"}
|
||||||
|
assert all(
|
||||||
|
operation["result"]["connection_lease_reused"] is True for operation in open_operations
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_control_session_recovers_changed_bridge_address_without_wifi_write(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, _ = service_with_fake_runtime(tmp_path)
|
||||||
|
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
||||||
|
service._connection_mode = "bridge" # noqa: SLF001
|
||||||
|
service._k1_ip = "10.255.254.54" # noqa: SLF001
|
||||||
|
service._device_id = "known-k1" # noqa: SLF001
|
||||||
|
service._device_session_id = "old-session" # noqa: SLF001
|
||||||
|
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||||
|
opened_hosts: list[str] = []
|
||||||
|
status_calls: list[dict[str, object]] = []
|
||||||
|
|
||||||
|
class FakeOpenControlSession:
|
||||||
|
def snapshot(self) -> dict[str, object]:
|
||||||
|
return {"state": "idle", "can_confirm_standby": False}
|
||||||
|
|
||||||
|
def open(self, *, host: str, **_: object) -> dict[str, object]:
|
||||||
|
opened_hosts.append(host)
|
||||||
|
return self.snapshot()
|
||||||
|
|
||||||
|
async def fake_status_read(*_: object, **kwargs: object) -> dict[str, Any]:
|
||||||
|
status_calls.append(kwargs)
|
||||||
|
return _wifi_status_read("10.255.254.77")
|
||||||
|
|
||||||
|
service._application_control_session = FakeOpenControlSession() # type: ignore[assignment] # noqa: SLF001
|
||||||
|
monkeypatch.setattr(facade_module, "read_wifi_status_once", fake_status_read)
|
||||||
|
monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False)
|
||||||
|
monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
facade_module,
|
||||||
|
"_control_endpoint_reachable",
|
||||||
|
lambda target: target == "10.255.254.77",
|
||||||
|
)
|
||||||
|
|
||||||
|
state = service.open_application_control_session(
|
||||||
|
OpenApplicationControlSessionRequest(
|
||||||
|
operator_present=True,
|
||||||
|
owner_controlled_device=True,
|
||||||
|
lixelgo_closed=True,
|
||||||
|
battery_storage_confirmed=True,
|
||||||
|
expected_physical_state_confirmed=True,
|
||||||
|
timezone_name="Europe/Moscow",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert status_calls == [{"timeout_seconds": 20.0, "rediscover": True}]
|
||||||
assert opened_hosts == ["10.255.254.77"]
|
assert opened_hosts == ["10.255.254.77"]
|
||||||
assert state["k1_ip"] == "10.255.254.77"
|
assert state["k1_ip"] == "10.255.254.77"
|
||||||
|
assert state["device_session"]["device_session_id"] != "old-session"
|
||||||
|
assert state["connection_verification"]["status"] == "recovered"
|
||||||
|
assert state["connection_verification"]["write_performed"] is False
|
||||||
|
open_operation = next(
|
||||||
|
operation
|
||||||
|
for operation in state["operations"]
|
||||||
|
if operation["action"] == "application-control.session.open"
|
||||||
|
)
|
||||||
|
assert open_operation["result"]["recovery_performed"] is True
|
||||||
|
assert open_operation["result"]["address_changed"] is True
|
||||||
|
assert open_operation["result"]["device_write_performed"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_control_session_prestart_failure_is_journaled_and_marks_lease_offline(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, _ = service_with_fake_runtime(tmp_path)
|
||||||
|
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
||||||
|
service._connection_mode = "bridge" # noqa: SLF001
|
||||||
|
service._k1_ip = "10.255.254.54" # noqa: SLF001
|
||||||
|
service._device_id = "known-k1" # noqa: SLF001
|
||||||
|
service._device_session_id = "known-session" # noqa: SLF001
|
||||||
|
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||||
|
|
||||||
|
async def failed_status_read(*_: object, **__: object) -> dict[str, Any]:
|
||||||
|
raise TimeoutError("synthetic BLE recovery timeout")
|
||||||
|
|
||||||
|
monkeypatch.setattr(facade_module, "read_wifi_status_once", failed_status_read)
|
||||||
|
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: False)
|
||||||
|
monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed")
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
facade_module.ConnectionLeaseUnavailable,
|
||||||
|
match="повторное чтение состояния",
|
||||||
|
):
|
||||||
|
service.open_application_control_session(
|
||||||
|
OpenApplicationControlSessionRequest(
|
||||||
|
operator_present=True,
|
||||||
|
owner_controlled_device=True,
|
||||||
|
lixelgo_closed=True,
|
||||||
|
battery_storage_confirmed=True,
|
||||||
|
expected_physical_state_confirmed=True,
|
||||||
|
timezone_name="Europe/Moscow",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
state = service.state()
|
||||||
|
assert state["application_control_session"]["state"] == "idle"
|
||||||
|
assert state["device_session"]["connectivity"] == "offline"
|
||||||
|
assert state["connection_verification"]["lease_state"] == "disconnected"
|
||||||
|
open_operation = next(
|
||||||
|
operation
|
||||||
|
for operation in state["operations"]
|
||||||
|
if operation["action"] == "application-control.session.open"
|
||||||
|
)
|
||||||
|
assert open_operation["status"] == "failed"
|
||||||
|
assert open_operation["error"] == {
|
||||||
|
"category": "connection",
|
||||||
|
"code": "connection_lease_ble_recovery_failed",
|
||||||
|
"retryable": False,
|
||||||
|
"safe_to_retry": True,
|
||||||
|
"side_effect_status": "none",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_bridge_route_mismatch_stops_before_ble_recovery_and_vendor_commands(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, _ = service_with_fake_runtime(tmp_path)
|
||||||
|
service._selected_device_id = "test-ble-transport" # noqa: SLF001
|
||||||
|
service._connection_mode = "bridge" # noqa: SLF001
|
||||||
|
service._k1_ip = "192.168.68.50" # noqa: SLF001
|
||||||
|
service._device_id = "known-k1" # noqa: SLF001
|
||||||
|
service._device_session_id = "known-session" # noqa: SLF001
|
||||||
|
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||||
|
|
||||||
|
async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]:
|
||||||
|
raise AssertionError("host route mismatch must stop before BLE recovery")
|
||||||
|
|
||||||
|
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: False)
|
||||||
|
monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "tunnel")
|
||||||
|
monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
facade_module.ConnectionLeaseUnavailable,
|
||||||
|
match="другой локальной сети",
|
||||||
|
):
|
||||||
|
service.open_application_control_session(
|
||||||
|
OpenApplicationControlSessionRequest(
|
||||||
|
operator_present=True,
|
||||||
|
owner_controlled_device=True,
|
||||||
|
lixelgo_closed=True,
|
||||||
|
battery_storage_confirmed=True,
|
||||||
|
expected_physical_state_confirmed=True,
|
||||||
|
timezone_name="Europe/Moscow",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
state = service.state()
|
||||||
|
assert state["connection_verification"]["reason_code"] == (
|
||||||
|
"connection_lease_host_route_mismatch"
|
||||||
|
)
|
||||||
|
assert state["connection_verification"]["endpoint_validation"] == "host-route"
|
||||||
|
assert state["connection_verification"]["host_route_class"] == "tunnel"
|
||||||
|
assert state["application_control_session"]["state"] == "idle"
|
||||||
|
|
||||||
|
|
||||||
def test_prepare_creates_provisional_device_session_and_profiled_acquisition(
|
def test_prepare_creates_provisional_device_session_and_profiled_acquisition(
|
||||||
@@ -384,18 +621,24 @@ def test_acquisition_is_unbounded_by_default_and_accepts_ten_hour_hint() -> None
|
|||||||
|
|
||||||
|
|
||||||
def test_connection_modes_require_their_exact_topology_attestation() -> None:
|
def test_connection_modes_require_their_exact_topology_attestation() -> None:
|
||||||
assert ConnectRequest(
|
assert (
|
||||||
|
ConnectRequest(
|
||||||
device_id="synthetic-device",
|
device_id="synthetic-device",
|
||||||
connection_mode="quick-connect",
|
connection_mode="quick-connect",
|
||||||
compatibility_attestation=QUICK_CONNECT_ATTESTATION,
|
compatibility_attestation=QUICK_CONNECT_ATTESTATION,
|
||||||
).connection_mode == "quick-connect"
|
).connection_mode
|
||||||
assert ConnectRequest(
|
== "quick-connect"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
ConnectRequest(
|
||||||
device_id="synthetic-device",
|
device_id="synthetic-device",
|
||||||
ssid="synthetic-network",
|
ssid="synthetic-network",
|
||||||
password=SecretStr(PRIMARY_TEST_CREDENTIAL),
|
password=SecretStr(PRIMARY_TEST_CREDENTIAL),
|
||||||
connection_mode="direct-connect",
|
connection_mode="direct-connect",
|
||||||
compatibility_attestation=DIRECT_CONNECT_ATTESTATION,
|
compatibility_attestation=DIRECT_CONNECT_ATTESTATION,
|
||||||
).connection_mode == "direct-connect"
|
).connection_mode
|
||||||
|
== "direct-connect"
|
||||||
|
)
|
||||||
with pytest.raises(ValidationError):
|
with pytest.raises(ValidationError):
|
||||||
ConnectRequest(
|
ConnectRequest(
|
||||||
device_id="synthetic-device",
|
device_id="synthetic-device",
|
||||||
@@ -628,6 +871,91 @@ def test_plugin_commanded_acquisition_keeps_start_and_stop_as_explicit_actions(
|
|||||||
assert completed["acquisition"]["result"]["device_state"] == "ready"
|
assert completed["acquisition"]["result"]["device_state"] == "ready"
|
||||||
assert completed["last_operation"]["status"] == "succeeded"
|
assert completed["last_operation"]["status"] == "succeeded"
|
||||||
assert runtime.stop_calls == 1
|
assert runtime.stop_calls == 1
|
||||||
|
assert completed["live_perception_shadow"]["active"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_next_scan_retires_stale_terminal_live_perception_ingress_before_start(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, runtime = service_with_fake_runtime(tmp_path)
|
||||||
|
service.live_perception_ingress.begin_session("stale-completed-session")
|
||||||
|
prepared = service.prepare_acquisition(
|
||||||
|
PrepareAcquisitionRequest(
|
||||||
|
project_name=PROJECT_NAME,
|
||||||
|
host="192.168.1.20",
|
||||||
|
compatibility_attestation=ATTESTATION,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
state = service.start_acquisition(
|
||||||
|
StartAcquisitionRequest(acquisition_id=prepared["acquisition"]["acquisition_id"])
|
||||||
|
)
|
||||||
|
|
||||||
|
assert runtime.start_calls
|
||||||
|
assert state["acquisition"]["state"] == "starting"
|
||||||
|
assert state["live_perception_shadow"]["active"] is True
|
||||||
|
assert state["live_perception_shadow"]["session_id"] != "stale-completed-session"
|
||||||
|
|
||||||
|
|
||||||
|
def test_confirmed_scanning_activates_and_records_right_camera(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
service, runtime = service_with_fake_runtime(tmp_path)
|
||||||
|
control = FakeInteractiveControlSession()
|
||||||
|
service._application_control_session = control # type: ignore[assignment] # noqa: SLF001
|
||||||
|
service._k1_ip = "192.168.1.20" # noqa: SLF001
|
||||||
|
service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001
|
||||||
|
|
||||||
|
prepared = service.prepare_acquisition(
|
||||||
|
PrepareAcquisitionRequest(
|
||||||
|
project_name="TEST001",
|
||||||
|
host="192.168.1.20",
|
||||||
|
compatibility_attestation=ATTESTATION,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
acquisition_id = prepared["acquisition"]["acquisition_id"]
|
||||||
|
service.start_acquisition(
|
||||||
|
StartAcquisitionRequest(
|
||||||
|
acquisition_id=acquisition_id,
|
||||||
|
physical_acceptance=PHYSICAL_ACCEPTANCE,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
out_dir = service._acquisition_out_dir # noqa: SLF001
|
||||||
|
assert out_dir is not None
|
||||||
|
out_dir.mkdir(parents=True)
|
||||||
|
events: list[tuple[str, object]] = []
|
||||||
|
camera_state: dict[str, object] = {
|
||||||
|
"phase": "idle",
|
||||||
|
"active_source_id": None,
|
||||||
|
"recording": {"active": False},
|
||||||
|
}
|
||||||
|
|
||||||
|
def select_camera(source_id: str, target: str) -> dict[str, object]:
|
||||||
|
events.append(("select", (source_id, target)))
|
||||||
|
camera_state["phase"] = "selected"
|
||||||
|
camera_state["active_source_id"] = source_id
|
||||||
|
return dict(camera_state)
|
||||||
|
|
||||||
|
def start_recording(session_dir: Path) -> dict[str, object]:
|
||||||
|
events.append(("record", session_dir))
|
||||||
|
camera_state["recording"] = {"active": True}
|
||||||
|
camera_state["phase"] = "connecting"
|
||||||
|
return dict(camera_state)
|
||||||
|
|
||||||
|
monkeypatch.setattr(service.camera_preview, "snapshot", lambda: dict(camera_state))
|
||||||
|
monkeypatch.setattr(service.camera_preview, "select", select_camera)
|
||||||
|
monkeypatch.setattr(service.camera_preview, "start_recording", start_recording)
|
||||||
|
|
||||||
|
service._activate_default_acquisition_camera() # noqa: SLF001
|
||||||
|
|
||||||
|
assert events == [
|
||||||
|
("select", ("sensor.camera.right", "192.168.1.20")),
|
||||||
|
("record", out_dir),
|
||||||
|
]
|
||||||
|
assert camera_state["active_source_id"] == "sensor.camera.right"
|
||||||
|
assert camera_state["recording"] == {"active": True}
|
||||||
|
assert runtime.source_mode == "live"
|
||||||
|
|
||||||
|
|
||||||
def test_device_standby_retires_sources_after_terminal_local_stop_failure(
|
def test_device_standby_retires_sources_after_terminal_local_stop_failure(
|
||||||
@@ -1611,6 +1939,9 @@ def test_runtime_error_terminalizes_pending_graceful_stop(tmp_path: Path) -> Non
|
|||||||
assert failed["acquisition"]["state"] == "failed"
|
assert failed["acquisition"]["state"] == "failed"
|
||||||
assert stop_operation["status"] == "failed"
|
assert stop_operation["status"] == "failed"
|
||||||
assert stop_operation["error"]["side_effect_status"] == "unknown"
|
assert stop_operation["error"]["side_effect_status"] == "unknown"
|
||||||
|
assert failed["source_mode"] == "idle"
|
||||||
|
assert failed["phase"] != "error"
|
||||||
|
assert runtime.stop_calls == 1
|
||||||
|
|
||||||
|
|
||||||
def test_receiver_completion_terminalizes_unconfirmed_graceful_stop(
|
def test_receiver_completion_terminalizes_unconfirmed_graceful_stop(
|
||||||
@@ -1970,6 +2301,11 @@ def test_network_provisioning_is_single_flight_and_secret_is_unwrapped_only_at_b
|
|||||||
}
|
}
|
||||||
|
|
||||||
monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision)
|
monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
facade_module,
|
||||||
|
"_host_route_class",
|
||||||
|
lambda _target: "direct-or-routed",
|
||||||
|
)
|
||||||
first = asyncio.create_task(
|
first = asyncio.create_task(
|
||||||
service.connect(
|
service.connect(
|
||||||
ConnectRequest(
|
ConnectRequest(
|
||||||
@@ -2005,6 +2341,117 @@ def test_network_provisioning_is_single_flight_and_secret_is_unwrapped_only_at_b
|
|||||||
assert {item["status"] for item in provision_operations} == {"succeeded", "failed"}
|
assert {item["status"] for item in provision_operations} == {"succeeded", "failed"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_bridge_network_change_retires_terminal_acquisition_and_receiver_error(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, runtime = service_with_fake_runtime(tmp_path)
|
||||||
|
service._devices = [{"device_id": "k1-a", "name": "XGR-TEST-A"}] # noqa: SLF001
|
||||||
|
prepared = service.prepare_acquisition(
|
||||||
|
PrepareAcquisitionRequest(
|
||||||
|
project_name=PROJECT_NAME,
|
||||||
|
host="192.168.1.20",
|
||||||
|
compatibility_attestation=ATTESTATION,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
service.abort_acquisition(
|
||||||
|
AbortAcquisitionRequest(
|
||||||
|
acquisition_id=prepared["acquisition"]["acquisition_id"],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
runtime.phase = "error"
|
||||||
|
runtime.source_mode = "live"
|
||||||
|
stop_calls_before_connect = runtime.stop_calls
|
||||||
|
|
||||||
|
async def fake_provision(*_: object, **__: object) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"started_at_utc": "2026-07-28T18:11:37Z",
|
||||||
|
"completed_at_utc": "2026-07-28T18:11:44Z",
|
||||||
|
"profile_id": "xgrids-k1-fw3-wifi-v1",
|
||||||
|
"outcome": "lan_address_observed",
|
||||||
|
"observations": [{"status": {"ipv4": "192.168.1.20"}}],
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
facade_module,
|
||||||
|
"_host_route_class",
|
||||||
|
lambda _target: "direct-or-routed",
|
||||||
|
)
|
||||||
|
connected = asyncio.run(
|
||||||
|
service.connect(
|
||||||
|
ConnectRequest(
|
||||||
|
device_id="k1-a",
|
||||||
|
ssid="lab-network",
|
||||||
|
password=SecretStr(PRIMARY_TEST_CREDENTIAL),
|
||||||
|
connection_mode="bridge",
|
||||||
|
compatibility_attestation=ATTESTATION,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert connected["connection_mode"] == "bridge"
|
||||||
|
assert connected["k1_ip"] == "192.168.1.20"
|
||||||
|
assert connected["acquisition"] is None
|
||||||
|
assert connected["source_mode"] == "idle"
|
||||||
|
assert connected["phase"] == "connected"
|
||||||
|
assert runtime.stop_calls == stop_calls_before_connect + 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_bridge_provisioning_reports_host_route_mismatch_without_hiding_device_success(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, _ = service_with_fake_runtime(tmp_path)
|
||||||
|
service._devices = [{"device_id": "k1-a", "name": "XGR-TEST-A"}] # noqa: SLF001
|
||||||
|
|
||||||
|
async def fake_provision(*_: object, **__: object) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"started_at_utc": "2026-07-28T19:43:03Z",
|
||||||
|
"completed_at_utc": "2026-07-28T19:43:16Z",
|
||||||
|
"profile_id": "xgrids-k1-fw3-wifi-v1",
|
||||||
|
"outcome": "lan_address_observed",
|
||||||
|
"observations": [{"status": {"ipv4": "192.168.68.50"}}],
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision)
|
||||||
|
monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "tunnel")
|
||||||
|
|
||||||
|
connected = asyncio.run(
|
||||||
|
service.connect(
|
||||||
|
ConnectRequest(
|
||||||
|
device_id="k1-a",
|
||||||
|
ssid="lab-router",
|
||||||
|
password=SecretStr(PRIMARY_TEST_CREDENTIAL),
|
||||||
|
connection_mode="bridge",
|
||||||
|
compatibility_attestation=ATTESTATION,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert connected["k1_ip"] == "192.168.68.50"
|
||||||
|
assert connected["phase"] == "device_selected"
|
||||||
|
assert "компьютер подключён к другой сети" in connected["message"]
|
||||||
|
assert connected["device_session"]["connectivity"] == "offline"
|
||||||
|
assert connected["connection_verification"] == {
|
||||||
|
"status": "host-route-mismatch",
|
||||||
|
"lease_state": "disconnected",
|
||||||
|
"lease_generation": 1,
|
||||||
|
"endpoint_validation": "host-route",
|
||||||
|
"network_reachability": "unreachable",
|
||||||
|
"reason_code": "connection_lease_host_route_mismatch",
|
||||||
|
"host_route_class": "tunnel",
|
||||||
|
"write_performed": True,
|
||||||
|
"observed_at": connected["connection_verification"]["observed_at"],
|
||||||
|
}
|
||||||
|
operation = next(
|
||||||
|
item for item in connected["operations"] if item["action"] == "network.provision"
|
||||||
|
)
|
||||||
|
assert operation["status"] == "succeeded"
|
||||||
|
assert operation["result"]["host_route_ready"] is False
|
||||||
|
assert operation["result"]["host_route_class"] == "tunnel"
|
||||||
|
|
||||||
|
|
||||||
def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
@@ -2028,9 +2475,7 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
|||||||
)
|
)
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def fake_activation_session(
|
async def fake_activation_session(device_id: str, **_: object) -> AsyncIterator[dict[str, Any]]:
|
||||||
device_id: str, **_: object
|
|
||||||
) -> AsyncIterator[dict[str, Any]]:
|
|
||||||
nonlocal ble_session_open
|
nonlocal ble_session_open
|
||||||
activation_calls.append(device_id)
|
activation_calls.append(device_id)
|
||||||
ble_session_open = True
|
ble_session_open = True
|
||||||
@@ -2092,9 +2537,7 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
|||||||
assert not ble_session_open
|
assert not ble_session_open
|
||||||
assert len(association_calls) == 1
|
assert len(association_calls) == 1
|
||||||
assert association_calls[0][0].name == "associate_wifi.swift"
|
assert association_calls[0][0].name == "associate_wifi.swift"
|
||||||
assert association_calls[0][1] == facade_module.quick_connect_host_profile_id(
|
assert association_calls[0][1] == facade_module.quick_connect_host_profile_id("XGR-TEST-A")
|
||||||
"XGR-TEST-A"
|
|
||||||
)
|
|
||||||
assert association_calls[0][2] == "XGR-TEST-A"
|
assert association_calls[0][2] == "XGR-TEST-A"
|
||||||
assert state["connection_mode"] == "quick-connect"
|
assert state["connection_mode"] == "quick-connect"
|
||||||
assert state["k1_ip"] == "192.168.56.1"
|
assert state["k1_ip"] == "192.168.56.1"
|
||||||
@@ -2102,9 +2545,7 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
|||||||
quick_sessions = sorted(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
quick_sessions = sorted(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
||||||
assert len(quick_sessions) == 1
|
assert len(quick_sessions) == 1
|
||||||
assert not (quick_sessions[0] / "provisioning.sensitive.json").exists()
|
assert not (quick_sessions[0] / "provisioning.sensitive.json").exists()
|
||||||
redacted_manifest = (quick_sessions[0] / "manifest.redacted.json").read_text(
|
redacted_manifest = (quick_sessions[0] / "manifest.redacted.json").read_text(encoding="utf-8")
|
||||||
encoding="utf-8"
|
|
||||||
)
|
|
||||||
assert PRIMARY_TEST_CREDENTIAL not in redacted_manifest
|
assert PRIMARY_TEST_CREDENTIAL not in redacted_manifest
|
||||||
assert "host_wifi_profile_id" in redacted_manifest
|
assert "host_wifi_profile_id" in redacted_manifest
|
||||||
assert '"host_wifi_profile_ready_before_device_write": true' in redacted_manifest
|
assert '"host_wifi_profile_ready_before_device_write": true' in redacted_manifest
|
||||||
@@ -2112,9 +2553,12 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
|||||||
assert "credential_provider_id" in redacted_manifest
|
assert "credential_provider_id" in redacted_manifest
|
||||||
assert "device_ap_activation_profile_id" in redacted_manifest
|
assert "device_ap_activation_profile_id" in redacted_manifest
|
||||||
assert (quick_sessions[0] / "ap-activation.redacted.json").exists()
|
assert (quick_sessions[0] / "ap-activation.redacted.json").exists()
|
||||||
assert service._camera_target_for_session( # noqa: SLF001
|
assert (
|
||||||
|
service._camera_target_for_session( # noqa: SLF001
|
||||||
state["device_session"]["device_session_id"]
|
state["device_session"]["device_session_id"]
|
||||||
) == "192.168.56.1"
|
)
|
||||||
|
== "192.168.56.1"
|
||||||
|
)
|
||||||
|
|
||||||
prepared = service.prepare_acquisition(
|
prepared = service.prepare_acquisition(
|
||||||
PrepareAcquisitionRequest(
|
PrepareAcquisitionRequest(
|
||||||
@@ -2158,6 +2602,11 @@ def test_direct_connect_reuses_the_reviewed_ble_provisioning_frame(
|
|||||||
forbidden_host_association,
|
forbidden_host_association,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False)
|
monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
facade_module,
|
||||||
|
"_host_route_class",
|
||||||
|
lambda _target: "direct-or-routed",
|
||||||
|
)
|
||||||
|
|
||||||
state = asyncio.run(
|
state = asyncio.run(
|
||||||
service.connect(
|
service.connect(
|
||||||
@@ -2171,14 +2620,10 @@ def test_direct_connect_reuses_the_reviewed_ble_provisioning_frame(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
assert provisioning_calls == [
|
assert provisioning_calls == [("k1-a", "controller-hotspot", PRIMARY_TEST_CREDENTIAL)]
|
||||||
("k1-a", "controller-hotspot", PRIMARY_TEST_CREDENTIAL)
|
|
||||||
]
|
|
||||||
assert state["connection_mode"] == "direct-connect"
|
assert state["connection_mode"] == "direct-connect"
|
||||||
assert state["k1_ip"] == "172.20.10.2"
|
assert state["k1_ip"] == "172.20.10.2"
|
||||||
assert state["compatibility"]["attestation"]["topology"] == (
|
assert state["compatibility"]["attestation"]["topology"] == ("controller-hotspot")
|
||||||
"controller-hotspot"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_quick_connect_missing_credential_provider_stops_before_ap_write(
|
def test_quick_connect_missing_credential_provider_stops_before_ap_write(
|
||||||
@@ -2232,9 +2677,7 @@ def test_quick_connect_missing_credential_provider_stops_before_ap_write(
|
|||||||
assert state["k1_ip"] == "192.168.1.20"
|
assert state["k1_ip"] == "192.168.1.20"
|
||||||
assert state["connection_mode"] == "bridge"
|
assert state["connection_mode"] == "bridge"
|
||||||
assert not list(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
assert not list(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
||||||
operation = next(
|
operation = next(item for item in state["operations"] if item["action"] == "network.provision")
|
||||||
item for item in state["operations"] if item["action"] == "network.provision"
|
|
||||||
)
|
|
||||||
assert operation["status"] == "failed"
|
assert operation["status"] == "failed"
|
||||||
assert operation["error"]["side_effect_status"] == "none"
|
assert operation["error"]["side_effect_status"] == "none"
|
||||||
assert operation["error"]["safe_to_retry"] is True
|
assert operation["error"]["safe_to_retry"] is True
|
||||||
@@ -2259,9 +2702,7 @@ def test_quick_connect_does_not_start_host_wifi_without_ap_ready_flag(
|
|||||||
)
|
)
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def not_ready_session(
|
async def not_ready_session(*_: object, **__: object) -> AsyncIterator[dict[str, Any]]:
|
||||||
*_: object, **__: object
|
|
||||||
) -> AsyncIterator[dict[str, Any]]:
|
|
||||||
yield {
|
yield {
|
||||||
"profile_id": "xgrids-k1-fw3-quick-connect-ap-v1",
|
"profile_id": "xgrids-k1-fw3-quick-connect-ap-v1",
|
||||||
"started_at_utc": "2026-07-19T15:00:00Z",
|
"started_at_utc": "2026-07-19T15:00:00Z",
|
||||||
@@ -2325,9 +2766,7 @@ def test_failed_connection_change_revokes_the_previous_route(
|
|||||||
)
|
)
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def fake_activation_session(
|
async def fake_activation_session(*_: object, **__: object) -> AsyncIterator[dict[str, Any]]:
|
||||||
*_: object, **__: object
|
|
||||||
) -> AsyncIterator[dict[str, Any]]:
|
|
||||||
yield {
|
yield {
|
||||||
"profile_id": "xgrids-k1-fw3-quick-connect-ap-v1",
|
"profile_id": "xgrids-k1-fw3-quick-connect-ap-v1",
|
||||||
"started_at_utc": "2026-07-19T15:00:00Z",
|
"started_at_utc": "2026-07-19T15:00:00Z",
|
||||||
@@ -2370,9 +2809,7 @@ def test_failed_connection_change_revokes_the_previous_route(
|
|||||||
assert state["compatibility"]["attestation"] is None
|
assert state["compatibility"]["attestation"] is None
|
||||||
quick_sessions = sorted(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
quick_sessions = sorted(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
||||||
failure_evidence = json.loads(
|
failure_evidence = json.loads(
|
||||||
(quick_sessions[0] / "host-wifi-association.redacted.json").read_text(
|
(quick_sessions[0] / "host-wifi-association.redacted.json").read_text(encoding="utf-8")
|
||||||
encoding="utf-8"
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
assert failure_evidence["reason_code"] == "network-not-found"
|
assert failure_evidence["reason_code"] == "network-not-found"
|
||||||
assert failure_evidence["scan_attempt_count"] == 4
|
assert failure_evidence["scan_attempt_count"] == 4
|
||||||
@@ -2413,9 +2850,7 @@ def test_network_provisioning_rejects_an_ipv4_owned_by_the_local_host(
|
|||||||
state = service.state()
|
state = service.state()
|
||||||
assert state["selected_device_id"] is None
|
assert state["selected_device_id"] is None
|
||||||
assert state["k1_ip"] is None
|
assert state["k1_ip"] is None
|
||||||
operation = next(
|
operation = next(item for item in state["operations"] if item["action"] == "network.provision")
|
||||||
item for item in state["operations"] if item["action"] == "network.provision"
|
|
||||||
)
|
|
||||||
assert operation["status"] == "failed"
|
assert operation["status"] == "failed"
|
||||||
assert operation["error"]["safe_to_retry"] is False
|
assert operation["error"]["safe_to_retry"] is False
|
||||||
assert operation["error"]["side_effect_status"] == "unknown"
|
assert operation["error"]["side_effect_status"] == "unknown"
|
||||||
|
|||||||
@@ -778,3 +778,36 @@ def test_post_publish_timeout_poisoned_transport_never_retries() -> None:
|
|||||||
required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"},
|
required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"},
|
||||||
)
|
)
|
||||||
assert len(fake.publish_calls) == 1
|
assert len(fake.publish_calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_network_loop_failure_keeps_exact_paho_result_and_phase() -> None:
|
||||||
|
class LoopFailureClient(FakeControlClient):
|
||||||
|
def loop(self, timeout: float) -> mqtt.MQTTErrorCode:
|
||||||
|
if self.publish_calls and not self.events:
|
||||||
|
return mqtt.MQTT_ERR_CONN_LOST
|
||||||
|
return super().loop(timeout)
|
||||||
|
|
||||||
|
fake = LoopFailureClient()
|
||||||
|
transport = ReviewedApplicationMqttTransport(
|
||||||
|
"192.168.1.20",
|
||||||
|
client_factory=lambda: cast(mqtt.Client, fake),
|
||||||
|
)
|
||||||
|
transport.open()
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
ApplicationCommandOutcomeUnknown,
|
||||||
|
match=r"phase=post-publish-drain, result=7: The connection was lost",
|
||||||
|
):
|
||||||
|
transport.exchange_batch_once(
|
||||||
|
[_envelope()],
|
||||||
|
required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"},
|
||||||
|
)
|
||||||
|
|
||||||
|
snapshot = transport.snapshot().as_dict()
|
||||||
|
assert snapshot["state"] == "poisoned"
|
||||||
|
assert snapshot["last_loop_result_code"] == int(mqtt.MQTT_ERR_CONN_LOST)
|
||||||
|
assert snapshot["last_loop_result_name"] == mqtt.error_string(
|
||||||
|
int(mqtt.MQTT_ERR_CONN_LOST)
|
||||||
|
)
|
||||||
|
assert snapshot["last_loop_phase"] == "post-publish-drain"
|
||||||
|
assert len(fake.publish_calls) == 1
|
||||||
|
|||||||
@@ -170,6 +170,7 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not(
|
|||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
FakeExecutor.records = []
|
FakeExecutor.records = []
|
||||||
|
scanning_observed = threading.Event()
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
session_module,
|
session_module,
|
||||||
"PhysicalAcceptanceDialogueExecutor",
|
"PhysicalAcceptanceDialogueExecutor",
|
||||||
@@ -181,6 +182,7 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not(
|
|||||||
loader,
|
loader,
|
||||||
transport_factory=lambda _host: transport, # type: ignore[arg-type]
|
transport_factory=lambda _host: transport, # type: ignore[arg-type]
|
||||||
epoch_seconds=lambda: 1_752_680_000,
|
epoch_seconds=lambda: 1_752_680_000,
|
||||||
|
scanning_observer=scanning_observed.set,
|
||||||
)
|
)
|
||||||
|
|
||||||
session.open(
|
session.open(
|
||||||
@@ -202,6 +204,7 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not(
|
|||||||
|
|
||||||
session.request_start(project_name="TEST001", confirmation=_confirmation())
|
session.request_start(project_name="TEST001", confirmation=_confirmation())
|
||||||
_wait_phase(session, "scanning")
|
_wait_phase(session, "scanning")
|
||||||
|
assert scanning_observed.wait(timeout=1.0)
|
||||||
assert FakeExecutor.records[-1] == "wait:stop"
|
assert FakeExecutor.records[-1] == "wait:stop"
|
||||||
|
|
||||||
session.request_stop(confirmation=_confirmation())
|
session.request_stop(confirmation=_confirmation())
|
||||||
@@ -210,6 +213,7 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not(
|
|||||||
assert completed["pending_operator_action"] is None
|
assert completed["pending_operator_action"] is None
|
||||||
assert loader.calls == 1
|
assert loader.calls == 1
|
||||||
assert completed["automatic_retry"] is False
|
assert completed["automatic_retry"] is False
|
||||||
|
assert completed["scanning_observer_errors"] == 0
|
||||||
assert completed["scripted_transitions"] is False
|
assert completed["scripted_transitions"] is False
|
||||||
assert FakeExecutor.records == [
|
assert FakeExecutor.records == [
|
||||||
"connection:1-6",
|
"connection:1-6",
|
||||||
@@ -562,6 +566,210 @@ def test_start_outcome_unknown_blocks_reopen_even_when_transport_is_closed(
|
|||||||
assert failed["can_open"] is False
|
assert failed["can_open"] is False
|
||||||
assert failed["failure"]["modeling_command_attempted"] is True # type: ignore[index]
|
assert failed["failure"]["modeling_command_attempted"] is True # type: ignore[index]
|
||||||
assert failed["failure"]["safe_to_retry"] is False # type: ignore[index]
|
assert failed["failure"]["safe_to_retry"] is False # type: ignore[index]
|
||||||
|
with pytest.raises(
|
||||||
|
session_module.ApplicationAcceptanceError,
|
||||||
|
match="cannot be retired",
|
||||||
|
):
|
||||||
|
session.retire_for_network_change()
|
||||||
|
|
||||||
|
|
||||||
|
def test_acknowledged_stop_disconnect_allows_only_explicit_network_change(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
@dataclass
|
||||||
|
class StopDisconnectTransportSnapshot:
|
||||||
|
state: str
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"state": self.state,
|
||||||
|
"publish_attempts": 15,
|
||||||
|
"qos2_completions": 15,
|
||||||
|
"correlated_responses": 13,
|
||||||
|
"ignored_known_responses": 304,
|
||||||
|
"late_known_responses": 0,
|
||||||
|
"device_status_reports": 52,
|
||||||
|
"latest_device_session_state": "scan_stopping",
|
||||||
|
"latest_device_project_bound": True,
|
||||||
|
"latest_system_error_code": None,
|
||||||
|
"last_loop_result_code": 7,
|
||||||
|
"last_loop_result_name": "The connection was lost.",
|
||||||
|
"last_loop_phase": "maintain-open",
|
||||||
|
"automatic_retry": False,
|
||||||
|
"automatic_reconnect": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
class StopDisconnectTransport(FakeTransport):
|
||||||
|
def snapshot(self) -> StopDisconnectTransportSnapshot:
|
||||||
|
return StopDisconnectTransportSnapshot(self.state)
|
||||||
|
|
||||||
|
class StopDisconnectExecutor(FakeExecutor):
|
||||||
|
def __init__(self, transport: StopDisconnectTransport) -> None:
|
||||||
|
super().__init__(transport)
|
||||||
|
self.stop_attempted = False
|
||||||
|
self.stop_complete = False
|
||||||
|
|
||||||
|
def execute_canonical_start(self, *_args: object, **_kwargs: object) -> object:
|
||||||
|
self.records.append("start:11-14")
|
||||||
|
return object()
|
||||||
|
|
||||||
|
def execute_canonical_stop(self, *_args: object, **_kwargs: object) -> object:
|
||||||
|
self.records.append("stop")
|
||||||
|
self.stop_attempted = True
|
||||||
|
self.stop_complete = True
|
||||||
|
return object()
|
||||||
|
|
||||||
|
def maintain_post_stop_until_standby(self) -> None:
|
||||||
|
self.records.append("wait:device-standby")
|
||||||
|
self.transport.state = "poisoned"
|
||||||
|
raise session_module.ApplicationCommandOutcomeUnknown(
|
||||||
|
"control MQTT network loop returned an error",
|
||||||
|
reason_code="mqtt_network_loop_failed",
|
||||||
|
)
|
||||||
|
|
||||||
|
def snapshot(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"dialogue_stage": (
|
||||||
|
"stop-acknowledged" if self.stop_complete else "test-stage"
|
||||||
|
),
|
||||||
|
"start_attempted": True,
|
||||||
|
"start_complete": True,
|
||||||
|
"stop_attempted": self.stop_attempted,
|
||||||
|
"stop_complete": self.stop_complete,
|
||||||
|
"response_evidence": [],
|
||||||
|
"automatic_retry": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
FakeExecutor.records = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
session_module,
|
||||||
|
"PhysicalAcceptanceDialogueExecutor",
|
||||||
|
StopDisconnectExecutor,
|
||||||
|
)
|
||||||
|
session = InteractiveApplicationControlSession(
|
||||||
|
FakeAuthorityLoader(),
|
||||||
|
transport_factory=lambda host: StopDisconnectTransport(host), # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
|
||||||
|
session.open(
|
||||||
|
host="192.168.1.20",
|
||||||
|
timezone_name="Europe/Moscow",
|
||||||
|
confirmation=_confirmation(),
|
||||||
|
)
|
||||||
|
_wait_phase(session, "connection-ready")
|
||||||
|
session.enter_workspace()
|
||||||
|
_wait_phase(session, "workspace-ready")
|
||||||
|
session.open_project_prompt()
|
||||||
|
_wait_phase(session, "project-ready")
|
||||||
|
session.request_start(project_name="TEST001", confirmation=_confirmation())
|
||||||
|
_wait_phase(session, "scanning")
|
||||||
|
session.request_stop(confirmation=_confirmation())
|
||||||
|
failed = _wait_phase(session, "failed")
|
||||||
|
failed_thread = session._thread # noqa: SLF001
|
||||||
|
assert failed_thread is not None
|
||||||
|
failed_thread.join(timeout=2.0)
|
||||||
|
assert not failed_thread.is_alive()
|
||||||
|
failed = session.snapshot()
|
||||||
|
|
||||||
|
assert failed["can_open"] is False
|
||||||
|
assert failed["outcome_unknown"] is True
|
||||||
|
failure = failed["failure"]
|
||||||
|
assert isinstance(failure, dict)
|
||||||
|
assert failure["safe_to_retry"] is False
|
||||||
|
assert failure["network_change_admissible"] is True
|
||||||
|
assert failure["network_change_reconciliation"] == {
|
||||||
|
"device_session_state": "scan_stopping",
|
||||||
|
"device_project_bound": True,
|
||||||
|
"system_error_code": None,
|
||||||
|
"stop_complete": True,
|
||||||
|
"standby_confirmed": False,
|
||||||
|
"decision": "explicit-network-change-only-after-acknowledged-stop",
|
||||||
|
"automatic_retry": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
retired = session.retire_for_network_change()
|
||||||
|
assert retired["state"] == "idle"
|
||||||
|
assert retired["failure"] is None
|
||||||
|
assert retired["can_open"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_prestart_loop_failure_allows_only_fresh_explicit_retry_after_ready_status(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
@dataclass
|
||||||
|
class ReconciledTransportSnapshot:
|
||||||
|
state: str
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"state": self.state,
|
||||||
|
"publish_attempts": 4,
|
||||||
|
"qos2_completions": 4,
|
||||||
|
"correlated_responses": 3,
|
||||||
|
"ignored_known_responses": 0,
|
||||||
|
"late_known_responses": 0,
|
||||||
|
"device_status_reports": 1,
|
||||||
|
"latest_device_session_state": "ready",
|
||||||
|
"latest_device_project_bound": False,
|
||||||
|
"latest_system_error_code": None,
|
||||||
|
"last_loop_result_code": 7,
|
||||||
|
"last_loop_result_name": "The connection was lost.",
|
||||||
|
"last_loop_phase": "post-publish-drain",
|
||||||
|
"automatic_retry": False,
|
||||||
|
"automatic_reconnect": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
class ReconciledTransport(FakeTransport):
|
||||||
|
def snapshot(self) -> ReconciledTransportSnapshot:
|
||||||
|
return ReconciledTransportSnapshot(self.state)
|
||||||
|
|
||||||
|
class PrestartLoopFailureExecutor(FakeExecutor):
|
||||||
|
def run_connection_stage(
|
||||||
|
self,
|
||||||
|
_orchestrator: object,
|
||||||
|
) -> LiveDeviceControlBinding:
|
||||||
|
raise session_module.ApplicationCommandOutcomeUnknown(
|
||||||
|
"control MQTT network loop returned an error",
|
||||||
|
reason_code="mqtt_network_loop_failed",
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
session_module,
|
||||||
|
"PhysicalAcceptanceDialogueExecutor",
|
||||||
|
PrestartLoopFailureExecutor,
|
||||||
|
)
|
||||||
|
session = InteractiveApplicationControlSession(
|
||||||
|
FakeAuthorityLoader(),
|
||||||
|
transport_factory=lambda host: ReconciledTransport(host), # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
|
||||||
|
session.open(
|
||||||
|
host="192.168.1.20",
|
||||||
|
timezone_name="Europe/Moscow",
|
||||||
|
confirmation=_confirmation(),
|
||||||
|
)
|
||||||
|
failed_thread = session._thread # noqa: SLF001
|
||||||
|
assert failed_thread is not None
|
||||||
|
failed_thread.join(timeout=2.0)
|
||||||
|
assert not failed_thread.is_alive()
|
||||||
|
failed = session.snapshot()
|
||||||
|
|
||||||
|
assert failed["state"] == "failed"
|
||||||
|
assert failed["outcome_unknown"] is True
|
||||||
|
assert failed["automatic_retry"] is False
|
||||||
|
assert failed["can_open"] is True
|
||||||
|
failure = failed["failure"]
|
||||||
|
assert isinstance(failure, dict)
|
||||||
|
assert failure["reason_code"] == "mqtt_network_loop_failed"
|
||||||
|
assert failure["modeling_command_attempted"] is False
|
||||||
|
assert failure["safe_to_retry"] is True
|
||||||
|
assert failure["status_reconciliation"] == {
|
||||||
|
"device_session_state": "ready",
|
||||||
|
"device_project_bound": False,
|
||||||
|
"system_error_code": None,
|
||||||
|
"decision": "safe-explicit-prestart-retry",
|
||||||
|
"automatic_retry": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_unavailable_transport_snapshot_after_publish_blocks_reopen(
|
def test_unavailable_transport_snapshot_after_publish_blocks_reopen(
|
||||||
|
|||||||
Reference in New Issue
Block a user